PostgreSQL Source Code git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
ts_typanalyze.c File Reference
#include "postgres.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_operator.h"
#include "commands/vacuum.h"
#include "common/hashfn.h"
#include "tsearch/ts_type.h"
#include "utils/builtins.h"
#include "varatt.h"
Include dependency graph for ts_typanalyze.c:

Go to the source code of this file.

Data Structures

struct  LexemeHashKey
 
struct  TrackItem
 

Functions

static void compute_tsvector_stats (VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int samplerows, double totalrows)
 
static void prune_lexemes_hashtable (HTAB *lexemes_tab, int b_current)
 
static uint32 lexeme_hash (const void *key, Size keysize)
 
static int lexeme_match (const void *key1, const void *key2, Size keysize)
 
static int lexeme_compare (const void *key1, const void *key2)
 
static int trackitem_compare_frequencies_desc (const void *e1, const void *e2, void *arg)
 
static int trackitem_compare_lexemes (const void *e1, const void *e2, void *arg)
 
Datum ts_typanalyze (PG_FUNCTION_ARGS)
 

Function Documentation

◆ compute_tsvector_stats()

static void compute_tsvector_stats ( VacAttrStats stats,
AnalyzeAttrFetchFunc  fetchfunc,
int  samplerows,
double  totalrows 
)
static

Definition at line 141 of file ts_typanalyze.c.

145{
146 int num_mcelem;
147 int null_cnt = 0;
148 double total_width = 0;
149
150 /* This is D from the LC algorithm. */
151 HTAB *lexemes_tab;
152 HASHCTL hash_ctl;
153 HASH_SEQ_STATUS scan_status;
154
155 /* This is the current bucket number from the LC algorithm */
156 int b_current;
157
158 /* This is 'w' from the LC algorithm */
159 int bucket_width;
160 int vector_no,
161 lexeme_no;
163
164 /*
165 * We want statistics_target * 10 lexemes in the MCELEM array. This
166 * multiplier is pretty arbitrary, but is meant to reflect the fact that
167 * the number of individual lexeme values tracked in pg_statistic ought to
168 * be more than the number of values for a simple scalar column.
169 */
170 num_mcelem = stats->attstattarget * 10;
171
172 /*
173 * We set bucket width equal to (num_mcelem + 10) / 0.007 as per the
174 * comment above.
175 */
176 bucket_width = (num_mcelem + 10) * 1000 / 7;
177
178 /*
179 * Create the hashtable. It will be in local memory, so we don't need to
180 * worry about overflowing the initial size. Also we don't need to pay any
181 * attention to locking and memory management.
182 */
183 hash_ctl.keysize = sizeof(LexemeHashKey);
184 hash_ctl.entrysize = sizeof(TrackItem);
185 hash_ctl.hash = lexeme_hash;
186 hash_ctl.match = lexeme_match;
187 hash_ctl.hcxt = CurrentMemoryContext;
188 lexemes_tab = hash_create("Analyzed lexemes table",
189 num_mcelem,
190 &hash_ctl,
192
193 /* Initialize counters. */
194 b_current = 1;
195 lexeme_no = 0;
196
197 /* Loop over the tsvectors. */
198 for (vector_no = 0; vector_no < samplerows; vector_no++)
199 {
200 Datum value;
201 bool isnull;
202 TSVector vector;
203 WordEntry *curentryptr;
204 char *lexemesptr;
205 int j;
206
208
209 value = fetchfunc(stats, vector_no, &isnull);
210
211 /*
212 * Check for null/nonnull.
213 */
214 if (isnull)
215 {
216 null_cnt++;
217 continue;
218 }
219
220 /*
221 * Add up widths for average-width calculation. Since it's a
222 * tsvector, we know it's varlena. As in the regular
223 * compute_minimal_stats function, we use the toasted width for this
224 * calculation.
225 */
226 total_width += VARSIZE_ANY(DatumGetPointer(value));
227
228 /*
229 * Now detoast the tsvector if needed.
230 */
231 vector = DatumGetTSVector(value);
232
233 /*
234 * We loop through the lexemes in the tsvector and add them to our
235 * tracking hashtable.
236 */
237 lexemesptr = STRPTR(vector);
238 curentryptr = ARRPTR(vector);
239 for (j = 0; j < vector->size; j++)
240 {
241 TrackItem *item;
242 bool found;
243
244 /*
245 * Construct a hash key. The key points into the (detoasted)
246 * tsvector value at this point, but if a new entry is created, we
247 * make a copy of it. This way we can free the tsvector value
248 * once we've processed all its lexemes.
249 */
250 hash_key.lexeme = lexemesptr + curentryptr->pos;
251 hash_key.length = curentryptr->len;
252
253 /* Lookup current lexeme in hashtable, adding it if new */
254 item = (TrackItem *) hash_search(lexemes_tab,
255 &hash_key,
256 HASH_ENTER, &found);
257
258 if (found)
259 {
260 /* The lexeme is already on the tracking list */
261 item->frequency++;
262 }
263 else
264 {
265 /* Initialize new tracking list element */
266 item->frequency = 1;
267 item->delta = b_current - 1;
268
269 item->key.lexeme = palloc(hash_key.length);
270 memcpy(item->key.lexeme, hash_key.lexeme, hash_key.length);
271 }
272
273 /* lexeme_no is the number of elements processed (ie N) */
274 lexeme_no++;
275
276 /* We prune the D structure after processing each bucket */
277 if (lexeme_no % bucket_width == 0)
278 {
279 prune_lexemes_hashtable(lexemes_tab, b_current);
280 b_current++;
281 }
282
283 /* Advance to the next WordEntry in the tsvector */
284 curentryptr++;
285 }
286
287 /* If the vector was toasted, free the detoasted copy. */
288 if (TSVectorGetDatum(vector) != value)
289 pfree(vector);
290 }
291
292 /* We can only compute real stats if we found some non-null values. */
293 if (null_cnt < samplerows)
294 {
295 int nonnull_cnt = samplerows - null_cnt;
296 int i;
297 TrackItem **sort_table;
298 TrackItem *item;
299 int track_len;
300 int cutoff_freq;
301 int minfreq,
302 maxfreq;
303
304 stats->stats_valid = true;
305 /* Do the simple null-frac and average width stats */
306 stats->stanullfrac = (double) null_cnt / (double) samplerows;
307 stats->stawidth = total_width / (double) nonnull_cnt;
308
309 /* Assume it's a unique column (see notes above) */
310 stats->stadistinct = -1.0 * (1.0 - stats->stanullfrac);
311
312 /*
313 * Construct an array of the interesting hashtable items, that is,
314 * those meeting the cutoff frequency (s - epsilon)*N. Also identify
315 * the minimum and maximum frequencies among these items.
316 *
317 * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
318 * frequency is 9*N / bucket_width.
319 */
320 cutoff_freq = 9 * lexeme_no / bucket_width;
321
322 i = hash_get_num_entries(lexemes_tab); /* surely enough space */
323 sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * i);
324
325 hash_seq_init(&scan_status, lexemes_tab);
326 track_len = 0;
327 minfreq = lexeme_no;
328 maxfreq = 0;
329 while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
330 {
331 if (item->frequency > cutoff_freq)
332 {
333 sort_table[track_len++] = item;
334 minfreq = Min(minfreq, item->frequency);
335 maxfreq = Max(maxfreq, item->frequency);
336 }
337 }
338 Assert(track_len <= i);
339
340 /* emit some statistics for debug purposes */
341 elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, "
342 "# lexemes = %d, hashtable size = %d, usable entries = %d",
343 num_mcelem, bucket_width, lexeme_no, i, track_len);
344
345 /*
346 * If we obtained more lexemes than we really want, get rid of those
347 * with least frequencies. The easiest way is to qsort the array into
348 * descending frequency order and truncate the array.
349 */
350 if (num_mcelem < track_len)
351 {
352 qsort_interruptible(sort_table, track_len, sizeof(TrackItem *),
354 /* reset minfreq to the smallest frequency we're keeping */
355 minfreq = sort_table[num_mcelem - 1]->frequency;
356 }
357 else
358 num_mcelem = track_len;
359
360 /* Generate MCELEM slot entry */
361 if (num_mcelem > 0)
362 {
363 MemoryContext old_context;
364 Datum *mcelem_values;
365 float4 *mcelem_freqs;
366
367 /*
368 * We want to store statistics sorted on the lexeme value using
369 * first length, then byte-for-byte comparison. The reason for
370 * doing length comparison first is that we don't care about the
371 * ordering so long as it's consistent, and comparing lengths
372 * first gives us a chance to avoid a strncmp() call.
373 *
374 * This is different from what we do with scalar statistics --
375 * they get sorted on frequencies. The rationale is that we
376 * usually search through most common elements looking for a
377 * specific value, so we can grab its frequency. When values are
378 * presorted we can employ binary search for that. See
379 * ts_selfuncs.c for a real usage scenario.
380 */
381 qsort_interruptible(sort_table, num_mcelem, sizeof(TrackItem *),
383
384 /* Must copy the target values into anl_context */
385 old_context = MemoryContextSwitchTo(stats->anl_context);
386
387 /*
388 * We sorted statistics on the lexeme value, but we want to be
389 * able to find out the minimal and maximal frequency without
390 * going through all the values. We keep those two extra
391 * frequencies in two extra cells in mcelem_freqs.
392 *
393 * (Note: the MCELEM statistics slot definition allows for a third
394 * extra number containing the frequency of nulls, but we don't
395 * create that for a tsvector column, since null elements aren't
396 * possible.)
397 */
398 mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
399 mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4));
400
401 /*
402 * See comments above about use of nonnull_cnt as the divisor for
403 * the final frequency estimates.
404 */
405 for (i = 0; i < num_mcelem; i++)
406 {
407 TrackItem *titem = sort_table[i];
408
409 mcelem_values[i] =
411 titem->key.length));
412 mcelem_freqs[i] = (double) titem->frequency / (double) nonnull_cnt;
413 }
414 mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
415 mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
416 MemoryContextSwitchTo(old_context);
417
418 stats->stakind[0] = STATISTIC_KIND_MCELEM;
419 stats->staop[0] = TextEqualOperator;
420 stats->stacoll[0] = DEFAULT_COLLATION_OID;
421 stats->stanumbers[0] = mcelem_freqs;
422 /* See above comment about two extra frequency fields */
423 stats->numnumbers[0] = num_mcelem + 2;
424 stats->stavalues[0] = mcelem_values;
425 stats->numvalues[0] = num_mcelem;
426 /* We are storing text values */
427 stats->statypid[0] = TEXTOID;
428 stats->statyplen[0] = -1; /* typlen, -1 for varlena */
429 stats->statypbyval[0] = false;
430 stats->statypalign[0] = 'i';
431 }
432 }
433 else
434 {
435 /* We found only nulls; assume the column is entirely null */
436 stats->stats_valid = true;
437 stats->stanullfrac = 1.0;
438 stats->stawidth = 0; /* "unknown" */
439 stats->stadistinct = 0.0; /* "unknown" */
440 }
441
442 /*
443 * We don't need to bother cleaning up any of our temporary palloc's. The
444 * hashtable should also go away, as it used a child memory context.
445 */
446}
#define Min(x, y)
Definition: c.h:961
#define Max(x, y)
Definition: c.h:955
#define Assert(condition)
Definition: c.h:815
float float4
Definition: c.h:586
#define ARRPTR(x)
Definition: cube.c:25
static dshash_hash hash_key(dshash_table *hash_table, const void *key)
Definition: dshash.c:1063
void * hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr)
Definition: dynahash.c:955
void * hash_seq_search(HASH_SEQ_STATUS *status)
Definition: dynahash.c:1420
long hash_get_num_entries(HTAB *hashp)
Definition: dynahash.c:1341
HTAB * hash_create(const char *tabname, long nelem, const HASHCTL *info, int flags)
Definition: dynahash.c:352
void hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp)
Definition: dynahash.c:1385
#define DEBUG3
Definition: elog.h:28
#define elog(elevel,...)
Definition: elog.h:225
@ HASH_ENTER
Definition: hsearch.h:114
#define HASH_CONTEXT
Definition: hsearch.h:102
#define HASH_ELEM
Definition: hsearch.h:95
#define HASH_COMPARE
Definition: hsearch.h:99
#define HASH_FUNCTION
Definition: hsearch.h:98
#define STRPTR(x)
Definition: hstore.h:76
static struct @162 value
int j
Definition: isn.c:73
int i
Definition: isn.c:72
void pfree(void *pointer)
Definition: mcxt.c:1521
void * palloc(Size size)
Definition: mcxt.c:1317
MemoryContext CurrentMemoryContext
Definition: mcxt.c:143
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:124
void qsort_interruptible(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:327
uintptr_t Datum
Definition: postgres.h:69
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:317
Size keysize
Definition: hsearch.h:75
HashValueFunc hash
Definition: hsearch.h:78
Size entrysize
Definition: hsearch.h:76
HashCompareFunc match
Definition: hsearch.h:80
MemoryContext hcxt
Definition: hsearch.h:86
Definition: dynahash.c:220
int32 size
Definition: ts_type.h:93
LexemeHashKey key
Definition: ts_typanalyze.c:35
bool stats_valid
Definition: vacuum.h:144
float4 stanullfrac
Definition: vacuum.h:145
int16 stakind[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:148
MemoryContext anl_context
Definition: vacuum.h:130
Oid statypid[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:162
Oid staop[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:149
Oid stacoll[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:150
char statypalign[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:165
float4 * stanumbers[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:152
int attstattarget
Definition: vacuum.h:125
int32 stawidth
Definition: vacuum.h:146
bool statypbyval[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:164
int16 statyplen[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:163
int numvalues[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:153
Datum * stavalues[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:154
float4 stadistinct
Definition: vacuum.h:147
int numnumbers[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:151
uint32 pos
Definition: ts_type.h:46
uint32 len
Definition: ts_type.h:45
static int trackitem_compare_frequencies_desc(const void *e1, const void *e2, void *arg)
static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
static int trackitem_compare_lexemes(const void *e1, const void *e2, void *arg)
static int lexeme_match(const void *key1, const void *key2, Size keysize)
static uint32 lexeme_hash(const void *key, Size keysize)
static TSVector DatumGetTSVector(Datum X)
Definition: ts_type.h:118
static Datum TSVectorGetDatum(const TSVectorData *X)
Definition: ts_type.h:130
void vacuum_delay_point(void)
Definition: vacuum.c:2361
#define VARSIZE_ANY(PTR)
Definition: varatt.h:311
text * cstring_to_text_with_len(const char *s, int len)
Definition: varlena.c:196

References VacAttrStats::anl_context, ARRPTR, Assert, VacAttrStats::attstattarget, cstring_to_text_with_len(), CurrentMemoryContext, DatumGetPointer(), DatumGetTSVector(), DEBUG3, TrackItem::delta, elog, HASHCTL::entrysize, TrackItem::frequency, HASHCTL::hash, HASH_COMPARE, HASH_CONTEXT, hash_create(), HASH_ELEM, HASH_ENTER, HASH_FUNCTION, hash_get_num_entries(), hash_key(), hash_search(), hash_seq_init(), hash_seq_search(), HASHCTL::hcxt, i, j, TrackItem::key, HASHCTL::keysize, WordEntry::len, LexemeHashKey::length, LexemeHashKey::lexeme, lexeme_hash(), lexeme_match(), HASHCTL::match, Max, MemoryContextSwitchTo(), Min, VacAttrStats::numnumbers, VacAttrStats::numvalues, palloc(), pfree(), PointerGetDatum(), WordEntry::pos, prune_lexemes_hashtable(), qsort_interruptible(), TSVectorData::size, VacAttrStats::stacoll, VacAttrStats::stadistinct, VacAttrStats::stakind, VacAttrStats::stanullfrac, VacAttrStats::stanumbers, VacAttrStats::staop, VacAttrStats::stats_valid, VacAttrStats::statypalign, VacAttrStats::statypbyval, VacAttrStats::statypid, VacAttrStats::statyplen, VacAttrStats::stavalues, VacAttrStats::stawidth, STRPTR, trackitem_compare_frequencies_desc(), trackitem_compare_lexemes(), TSVectorGetDatum(), vacuum_delay_point(), value, and VARSIZE_ANY.

Referenced by ts_typanalyze().

◆ lexeme_compare()

static int lexeme_compare ( const void *  key1,
const void *  key2 
)
static

Definition at line 500 of file ts_typanalyze.c.

501{
502 const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
503 const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
504
505 /* First, compare by length */
506 if (d1->length > d2->length)
507 return 1;
508 else if (d1->length < d2->length)
509 return -1;
510 /* Lengths are equal, do a byte-by-byte comparison */
511 return strncmp(d1->lexeme, d2->lexeme, d1->length);
512}

References LexemeHashKey::length, and LexemeHashKey::lexeme.

Referenced by lexeme_match(), and trackitem_compare_lexemes().

◆ lexeme_hash()

static uint32 lexeme_hash ( const void *  key,
Size  keysize 
)
static

Definition at line 478 of file ts_typanalyze.c.

479{
480 const LexemeHashKey *l = (const LexemeHashKey *) key;
481
482 return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
483 l->length));
484}
static Datum hash_any(const unsigned char *k, int keylen)
Definition: hashfn.h:31
static uint32 DatumGetUInt32(Datum X)
Definition: postgres.h:227

References DatumGetUInt32(), hash_any(), sort-test::key, LexemeHashKey::length, and LexemeHashKey::lexeme.

Referenced by compute_tsvector_stats().

◆ lexeme_match()

static int lexeme_match ( const void *  key1,
const void *  key2,
Size  keysize 
)
static

Definition at line 490 of file ts_typanalyze.c.

491{
492 /* The keysize parameter is superfluous, the keys store their lengths */
493 return lexeme_compare(key1, key2);
494}
static int lexeme_compare(const void *key1, const void *key2)

References lexeme_compare().

Referenced by compute_tsvector_stats().

◆ prune_lexemes_hashtable()

static void prune_lexemes_hashtable ( HTAB lexemes_tab,
int  b_current 
)
static

Definition at line 453 of file ts_typanalyze.c.

454{
455 HASH_SEQ_STATUS scan_status;
456 TrackItem *item;
457
458 hash_seq_init(&scan_status, lexemes_tab);
459 while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
460 {
461 if (item->frequency + item->delta <= b_current)
462 {
463 char *lexeme = item->key.lexeme;
464
465 if (hash_search(lexemes_tab, &item->key,
466 HASH_REMOVE, NULL) == NULL)
467 elog(ERROR, "hash table corrupted");
468 pfree(lexeme);
469 }
470 }
471}
#define ERROR
Definition: elog.h:39
@ HASH_REMOVE
Definition: hsearch.h:115

References TrackItem::delta, elog, ERROR, TrackItem::frequency, HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), TrackItem::key, LexemeHashKey::lexeme, and pfree().

Referenced by compute_tsvector_stats().

◆ trackitem_compare_frequencies_desc()

static int trackitem_compare_frequencies_desc ( const void *  e1,
const void *  e2,
void *  arg 
)
static

Definition at line 518 of file ts_typanalyze.c.

519{
520 const TrackItem *const *t1 = (const TrackItem *const *) e1;
521 const TrackItem *const *t2 = (const TrackItem *const *) e2;
522
523 return (*t2)->frequency - (*t1)->frequency;
524}

References TrackItem::frequency.

Referenced by compute_tsvector_stats().

◆ trackitem_compare_lexemes()

static int trackitem_compare_lexemes ( const void *  e1,
const void *  e2,
void *  arg 
)
static

Definition at line 530 of file ts_typanalyze.c.

531{
532 const TrackItem *const *t1 = (const TrackItem *const *) e1;
533 const TrackItem *const *t2 = (const TrackItem *const *) e2;
534
535 return lexeme_compare(&(*t1)->key, &(*t2)->key);
536}

References lexeme_compare().

Referenced by compute_tsvector_stats().

◆ ts_typanalyze()

Datum ts_typanalyze ( PG_FUNCTION_ARGS  )

Definition at line 58 of file ts_typanalyze.c.

59{
61
62 /* If the attstattarget column is negative, use the default value */
63 if (stats->attstattarget < 0)
65
67 /* see comment about the choice of minrows in commands/analyze.c */
68 stats->minrows = 300 * stats->attstattarget;
69
70 PG_RETURN_BOOL(true);
71}
int default_statistics_target
Definition: analyze.c:71
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:276
#define PG_RETURN_BOOL(x)
Definition: fmgr.h:359
int minrows
Definition: vacuum.h:137
AnalyzeAttrComputeStatsFunc compute_stats
Definition: vacuum.h:136
static void compute_tsvector_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int samplerows, double totalrows)

References VacAttrStats::attstattarget, VacAttrStats::compute_stats, compute_tsvector_stats(), default_statistics_target, VacAttrStats::minrows, PG_GETARG_POINTER, and PG_RETURN_BOOL.