ts_typanalyze.c File Reference

#include "postgres.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_operator.h"
#include "commands/vacuum.h"
#include "common/hashfn.h"
#include "tsearch/ts_type.h"
#include "utils/builtins.h"
#include "varatt.h"

Include dependency graph for ts_typanalyze.c:

Go to the source code of this file.

Data Structures
struct	LexemeHashKey
struct	TrackItem

Functions
static void	compute_tsvector_stats (VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int samplerows, double totalrows)
static void	prune_lexemes_hashtable (HTAB *lexemes_tab, int b_current)
static uint32	lexeme_hash (const void *key, Size keysize)
static int	lexeme_match (const void *key1, const void *key2, Size keysize)
static int	lexeme_compare (const void *key1, const void *key2)
static int	trackitem_compare_frequencies_desc (const void *e1, const void *e2, void *arg)
static int	trackitem_compare_lexemes (const void *e1, const void *e2, void *arg)
Datum	ts_typanalyze (PG_FUNCTION_ARGS)

Function Documentation

compute_tsvector_stats()

static void compute_tsvector_stats ( VacAttrStats *  stats, AnalyzeAttrFetchFunc  fetchfunc, int  samplerows, double  totalrows  )

static

Definition at line 141 of file ts_typanalyze.c.

{
    int         num_mcelem;
    int         null_cnt = 0;
    double      total_width = 0;
 
    /* This is D from the LC algorithm. */
    HTAB       *lexemes_tab;
    HASHCTL     hash_ctl;
    HASH_SEQ_STATUS scan_status;
 
    /* This is the current bucket number from the LC algorithm */
    int         b_current;
 
    /* This is 'w' from the LC algorithm */
    int         bucket_width;
    int         vector_no,
                lexeme_no;
    LexemeHashKey hash_key;
 
    /*
     * We want statistics_target * 10 lexemes in the MCELEM array.  This
     * multiplier is pretty arbitrary, but is meant to reflect the fact that
     * the number of individual lexeme values tracked in pg_statistic ought to
     * be more than the number of values for a simple scalar column.
     */
    num_mcelem = stats->attstattarget * 10;
 
    /*
     * We set bucket width equal to (num_mcelem + 10) / 0.007 as per the
     * comment above.
     */
    bucket_width = (num_mcelem + 10) * 1000 / 7;
 
    /*
     * Create the hashtable. It will be in local memory, so we don't need to
     * worry about overflowing the initial size. Also we don't need to pay any
     * attention to locking and memory management.
     */
    hash_ctl.keysize = sizeof(LexemeHashKey);
    hash_ctl.entrysize = sizeof(TrackItem);
    hash_ctl.hash = lexeme_hash;
    hash_ctl.match = lexeme_match;
    hash_ctl.hcxt = CurrentMemoryContext;
    lexemes_tab = hash_create("Analyzed lexemes table",
                              num_mcelem,
                              &hash_ctl,
                              HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
 
    /* Initialize counters. */
    b_current = 1;
    lexeme_no = 0;
 
    /* Loop over the tsvectors. */
    for (vector_no = 0; vector_no < samplerows; vector_no++)
    {
        Datum       value;
        bool        isnull;
        TSVector    vector;
        WordEntry  *curentryptr;
        char       *lexemesptr;
        int         j;
 
        vacuum_delay_point(true);
 
        value = fetchfunc(stats, vector_no, &isnull);
 
        /*
         * Check for null/nonnull.
         */
        if (isnull)
        {
            null_cnt++;
            continue;
        }
 
        /*
         * Add up widths for average-width calculation.  Since it's a
         * tsvector, we know it's varlena.  As in the regular
         * compute_minimal_stats function, we use the toasted width for this
         * calculation.
         */
        total_width += VARSIZE_ANY(DatumGetPointer(value));
 
        /*
         * Now detoast the tsvector if needed.
         */
        vector = DatumGetTSVector(value);
 
        /*
         * We loop through the lexemes in the tsvector and add them to our
         * tracking hashtable.
         */
        lexemesptr = STRPTR(vector);
        curentryptr = ARRPTR(vector);
        for (j = 0; j < vector->size; j++)
        {
            TrackItem  *item;
            bool        found;
 
            /*
             * Construct a hash key.  The key points into the (detoasted)
             * tsvector value at this point, but if a new entry is created, we
             * make a copy of it.  This way we can free the tsvector value
             * once we've processed all its lexemes.
             */
            hash_key.lexeme = lexemesptr + curentryptr->pos;
            hash_key.length = curentryptr->len;
 
            /* Lookup current lexeme in hashtable, adding it if new */
            item = (TrackItem *) hash_search(lexemes_tab,
                                             &hash_key,
                                             HASH_ENTER, &found);
 
            if (found)
            {
                /* The lexeme is already on the tracking list */
                item->frequency++;
            }
            else
            {
                /* Initialize new tracking list element */
                item->frequency = 1;
                item->delta = b_current - 1;
 
                item->key.lexeme = palloc(hash_key.length);
                memcpy(item->key.lexeme, hash_key.lexeme, hash_key.length);
            }
 
            /* lexeme_no is the number of elements processed (ie N) */
            lexeme_no++;
 
            /* We prune the D structure after processing each bucket */
            if (lexeme_no % bucket_width == 0)
            {
                prune_lexemes_hashtable(lexemes_tab, b_current);
                b_current++;
            }
 
            /* Advance to the next WordEntry in the tsvector */
            curentryptr++;
        }
 
        /* If the vector was toasted, free the detoasted copy. */
        if (TSVectorGetDatum(vector) != value)
            pfree(vector);
    }
 
    /* We can only compute real stats if we found some non-null values. */
    if (null_cnt < samplerows)
    {
        int         nonnull_cnt = samplerows - null_cnt;
        int         i;
        TrackItem **sort_table;
        TrackItem  *item;
        int         track_len;
        int         cutoff_freq;
        int         minfreq,
                    maxfreq;
 
        stats->stats_valid = true;
        /* Do the simple null-frac and average width stats */
        stats->stanullfrac = (double) null_cnt / (double) samplerows;
        stats->stawidth = total_width / (double) nonnull_cnt;
 
        /* Assume it's a unique column (see notes above) */
        stats->stadistinct = -1.0 * (1.0 - stats->stanullfrac);
 
        /*
         * Construct an array of the interesting hashtable items, that is,
         * those meeting the cutoff frequency (s - epsilon)*N.  Also identify
         * the maximum frequency among these items.
         *
         * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
         * frequency is 9*N / bucket_width.
         */
        cutoff_freq = 9 * lexeme_no / bucket_width;
 
        i = hash_get_num_entries(lexemes_tab);  /* surely enough space */
        sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * i);
 
        hash_seq_init(&scan_status, lexemes_tab);
        track_len = 0;
        maxfreq = 0;
        while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
        {
            if (item->frequency > cutoff_freq)
            {
                sort_table[track_len++] = item;
                maxfreq = Max(maxfreq, item->frequency);
            }
        }
        Assert(track_len <= i);
 
        /* emit some statistics for debug purposes */
        elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, "
             "# lexemes = %d, hashtable size = %d, usable entries = %d",
             num_mcelem, bucket_width, lexeme_no, i, track_len);
 
        /*
         * If we obtained more lexemes than we really want, get rid of those
         * with least frequencies.  The easiest way is to qsort the array into
         * descending frequency order and truncate the array.
         *
         * If we did not find more elements than we want, then it is safe to
         * assume that the stored MCE array will contain every element with
         * frequency above the cutoff.  In that case, rather than storing the
         * smallest frequency we are keeping, we want to store the minimum
         * frequency that would have been accepted as a valid MCE.  The
         * selectivity functions can assume that that is an upper bound on the
         * frequency of elements not present in the array.
         *
         * If we found no candidate MCEs at all, we still want to record the
         * cutoff frequency, since it's still valid to assume that no element
         * has frequency more than that.
         */
        if (num_mcelem < track_len)
        {
            qsort_interruptible(sort_table, track_len, sizeof(TrackItem *),
                                trackitem_compare_frequencies_desc, NULL);
            /* set minfreq to the smallest frequency we're keeping */
            minfreq = sort_table[num_mcelem - 1]->frequency;
        }
        else
        {
            num_mcelem = track_len;
            /* set minfreq to the minimum frequency above the cutoff */
            minfreq = cutoff_freq + 1;
            /* ensure maxfreq is nonzero, too */
            if (track_len == 0)
                maxfreq = minfreq;
        }
 
        /* Generate MCELEM slot entry */
        if (num_mcelem >= 0)
        {
            MemoryContext old_context;
            Datum      *mcelem_values;
            float4     *mcelem_freqs;
 
            /*
             * We want to store statistics sorted on the lexeme value using
             * first length, then byte-for-byte comparison. The reason for
             * doing length comparison first is that we don't care about the
             * ordering so long as it's consistent, and comparing lengths
             * first gives us a chance to avoid a strncmp() call.
             *
             * This is different from what we do with scalar statistics --
             * they get sorted on frequencies. The rationale is that we
             * usually search through most common elements looking for a
             * specific value, so we can grab its frequency.  When values are
             * presorted we can employ binary search for that.  See
             * ts_selfuncs.c for a real usage scenario.
             */
            qsort_interruptible(sort_table, num_mcelem, sizeof(TrackItem *),
                                trackitem_compare_lexemes, NULL);
 
            /* Must copy the target values into anl_context */
            old_context = MemoryContextSwitchTo(stats->anl_context);
 
            /*
             * We sorted statistics on the lexeme value, but we want to be
             * able to find out the minimal and maximal frequency without
             * going through all the values.  We keep those two extra
             * frequencies in two extra cells in mcelem_freqs.
             *
             * (Note: the MCELEM statistics slot definition allows for a third
             * extra number containing the frequency of nulls, but we don't
             * create that for a tsvector column, since null elements aren't
             * possible.)
             */
            mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
            mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4));
 
            /*
             * See comments above about use of nonnull_cnt as the divisor for
             * the final frequency estimates.
             */
            for (i = 0; i < num_mcelem; i++)
            {
                TrackItem  *titem = sort_table[i];
 
                mcelem_values[i] =
                    PointerGetDatum(cstring_to_text_with_len(titem->key.lexeme,
                                                             titem->key.length));
                mcelem_freqs[i] = (double) titem->frequency / (double) nonnull_cnt;
            }
            mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
            mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
            MemoryContextSwitchTo(old_context);
 
            stats->stakind[0] = STATISTIC_KIND_MCELEM;
            stats->staop[0] = TextEqualOperator;
            stats->stacoll[0] = DEFAULT_COLLATION_OID;
            stats->stanumbers[0] = mcelem_freqs;
            /* See above comment about two extra frequency fields */
            stats->numnumbers[0] = num_mcelem + 2;
            stats->stavalues[0] = mcelem_values;
            stats->numvalues[0] = num_mcelem;
            /* We are storing text values */
            stats->statypid[0] = TEXTOID;
            stats->statyplen[0] = -1;   /* typlen, -1 for varlena */
            stats->statypbyval[0] = false;
            stats->statypalign[0] = 'i';
        }
    }
    else
    {
        /* We found only nulls; assume the column is entirely null */
        stats->stats_valid = true;
        stats->stanullfrac = 1.0;
        stats->stawidth = 0;    /* "unknown" */
        stats->stadistinct = 0.0;   /* "unknown" */
    }
 
    /*
     * We don't need to bother cleaning up any of our temporary palloc's. The
     * hashtable should also go away, as it used a child memory context.
     */
}

References VacAttrStats::anl_context, ARRPTR, Assert(), VacAttrStats::attstattarget, cstring_to_text_with_len(), CurrentMemoryContext, DatumGetPointer(), DatumGetTSVector(), DEBUG3, TrackItem::delta, elog, HASHCTL::entrysize, TrackItem::frequency, HASHCTL::hash, HASH_COMPARE, HASH_CONTEXT, hash_create(), HASH_ELEM, HASH_ENTER, HASH_FUNCTION, hash_get_num_entries(), hash_key(), hash_search(), hash_seq_init(), hash_seq_search(), HASHCTL::hcxt, i, j, TrackItem::key, HASHCTL::keysize, WordEntry::len, LexemeHashKey::length, LexemeHashKey::lexeme, lexeme_hash(), lexeme_match(), HASHCTL::match, Max, MemoryContextSwitchTo(), VacAttrStats::numnumbers, VacAttrStats::numvalues, palloc(), pfree(), PointerGetDatum(), WordEntry::pos, prune_lexemes_hashtable(), qsort_interruptible(), TSVectorData::size, VacAttrStats::stacoll, VacAttrStats::stadistinct, VacAttrStats::stakind, VacAttrStats::stanullfrac, VacAttrStats::stanumbers, VacAttrStats::staop, VacAttrStats::stats_valid, VacAttrStats::statypalign, VacAttrStats::statypbyval, VacAttrStats::statypid, VacAttrStats::statyplen, VacAttrStats::stavalues, VacAttrStats::stawidth, STRPTR, trackitem_compare_frequencies_desc(), trackitem_compare_lexemes(), TSVectorGetDatum(), vacuum_delay_point(), value, and VARSIZE_ANY().

Referenced by ts_typanalyze().

lexeme_compare()

static int lexeme_compare ( const void *  key1, const void *  key2  )

static

Definition at line 517 of file ts_typanalyze.c.

{
    const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
    const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
 
    /* First, compare by length */
    if (d1->length > d2->length)
        return 1;
    else if (d1->length < d2->length)
        return -1;
    /* Lengths are equal, do a byte-by-byte comparison */
    return strncmp(d1->lexeme, d2->lexeme, d1->length);
}

References LexemeHashKey::length, and LexemeHashKey::lexeme.

Referenced by lexeme_match(), and trackitem_compare_lexemes().

lexeme_hash()

static uint32 lexeme_hash ( const void *  key, Size  keysize  )

static

Definition at line 495 of file ts_typanalyze.c.

{
    const LexemeHashKey *l = (const LexemeHashKey *) key;
 
    return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
                                   l->length));
}

References DatumGetUInt32(), hash_any(), sort-test::key, LexemeHashKey::length, and LexemeHashKey::lexeme.

Referenced by compute_tsvector_stats().

lexeme_match()

static int lexeme_match ( const void *  key1, const void *  key2, Size  keysize  )

static

Definition at line 507 of file ts_typanalyze.c.

{
    /* The keysize parameter is superfluous, the keys store their lengths */
    return lexeme_compare(key1, key2);
}

References lexeme_compare().

Referenced by compute_tsvector_stats().

prune_lexemes_hashtable()

static void prune_lexemes_hashtable ( HTAB *  lexemes_tab, int  b_current  )

static

Definition at line 470 of file ts_typanalyze.c.

{
    HASH_SEQ_STATUS scan_status;
    TrackItem  *item;
 
    hash_seq_init(&scan_status, lexemes_tab);
    while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
    {
        if (item->frequency + item->delta <= b_current)
        {
            char       *lexeme = item->key.lexeme;
 
            if (hash_search(lexemes_tab, &item->key,
                            HASH_REMOVE, NULL) == NULL)
                elog(ERROR, "hash table corrupted");
            pfree(lexeme);
        }
    }
}

References TrackItem::delta, elog, ERROR, TrackItem::frequency, HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), TrackItem::key, LexemeHashKey::lexeme, and pfree().

Referenced by compute_tsvector_stats().

trackitem_compare_frequencies_desc()

static int trackitem_compare_frequencies_desc ( const void *  e1, const void *  e2, void *  arg  )

static

Definition at line 535 of file ts_typanalyze.c.

{
    const TrackItem *const *t1 = (const TrackItem *const *) e1;
    const TrackItem *const *t2 = (const TrackItem *const *) e2;
 
    return (*t2)->frequency - (*t1)->frequency;
}

References TrackItem::frequency.

Referenced by compute_tsvector_stats().

trackitem_compare_lexemes()

static int trackitem_compare_lexemes ( const void *  e1, const void *  e2, void *  arg  )

static

Definition at line 547 of file ts_typanalyze.c.

{
    const TrackItem *const *t1 = (const TrackItem *const *) e1;
    const TrackItem *const *t2 = (const TrackItem *const *) e2;
 
    return lexeme_compare(&(*t1)->key, &(*t2)->key);
}

References lexeme_compare().

Referenced by compute_tsvector_stats().

ts_typanalyze()

Datum ts_typanalyze

(

PG_FUNCTION_ARGS

)

Definition at line 58 of file ts_typanalyze.c.

{
    VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
 
    /* If the attstattarget column is negative, use the default value */
    if (stats->attstattarget < 0)
        stats->attstattarget = default_statistics_target;
 
    stats->compute_stats = compute_tsvector_stats;
    /* see comment about the choice of minrows in commands/analyze.c */
    stats->minrows = 300 * stats->attstattarget;
 
    PG_RETURN_BOOL(true);
}

References VacAttrStats::attstattarget, VacAttrStats::compute_stats, compute_tsvector_stats(), default_statistics_target, VacAttrStats::minrows, PG_GETARG_POINTER, and PG_RETURN_BOOL.