#include "postgres.h"
#include "access/htup_details.h"
#include "access/parallel.h"
#include "catalog/objectaccess.h"
#include "catalog/pg_aggregate.h"
#include "catalog/pg_proc.h"
#include "catalog/pg_type.h"
#include "common/hashfn.h"
#include "executor/execExpr.h"
#include "executor/executor.h"
#include "executor/nodeAgg.h"
#include "lib/hyperloglog.h"
#include "miscadmin.h"
#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_agg.h"
#include "parser/parse_coerce.h"
#include "utils/acl.h"
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/dynahash.h"
#include "utils/expandeddatum.h"
#include "utils/injection_point.h"
#include "utils/logtape.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/memutils_memorychunk.h"
#include "utils/syscache.h"
#include "utils/tuplesort.h"

Include dependency graph for nodeAgg.c:

Data Structures
struct	HashAggSpill

struct	HashAggBatch

struct	FindColsContext

Macros
#define	HASHAGG_PARTITION_FACTOR 1.50

#define	HASHAGG_MIN_PARTITIONS 4

#define	HASHAGG_MAX_PARTITIONS 1024

#define	HASHAGG_READ_BUFFER_SIZE BLCKSZ

#define	HASHAGG_WRITE_BUFFER_SIZE BLCKSZ

#define	HASHAGG_HLL_BIT_WIDTH 5

#define	CHUNKHDRSZ sizeof(MemoryChunk)

Typedefs
typedef struct HashAggSpill	HashAggSpill

typedef struct HashAggBatch	HashAggBatch

typedef struct FindColsContext	FindColsContext

Functions
static void	select_current_set (AggState *aggstate, int setno, bool is_hash)

static void	initialize_phase (AggState *aggstate, int newphase)

static TupleTableSlot *	fetch_input_tuple (AggState *aggstate)

static void	initialize_aggregates (AggState aggstate, AggStatePerGroup pergroups, int numReset)

static void	advance_transition_function (AggState *aggstate, AggStatePerTrans pertrans, AggStatePerGroup pergroupstate)

static void	advance_aggregates (AggState *aggstate)

static void	process_ordered_aggregate_single (AggState *aggstate, AggStatePerTrans pertrans, AggStatePerGroup pergroupstate)

static void	process_ordered_aggregate_multi (AggState *aggstate, AggStatePerTrans pertrans, AggStatePerGroup pergroupstate)

static void	finalize_aggregate (AggState aggstate, AggStatePerAgg peragg, AggStatePerGroup pergroupstate, Datum resultVal, bool *resultIsNull)

static void	finalize_partialaggregate (AggState aggstate, AggStatePerAgg peragg, AggStatePerGroup pergroupstate, Datum resultVal, bool *resultIsNull)

static void	prepare_hash_slot (AggStatePerHash perhash, TupleTableSlot inputslot, TupleTableSlot hashslot)

static void	prepare_projection_slot (AggState aggstate, TupleTableSlot slot, int currentSet)

static void	finalize_aggregates (AggState *aggstate, AggStatePerAgg peraggs, AggStatePerGroup pergroup)

static TupleTableSlot *	project_aggregates (AggState *aggstate)

static void	find_cols (AggState aggstate, Bitmapset aggregated, Bitmapset *unaggregated)

static bool	find_cols_walker (Node node, FindColsContext context)

static void	build_hash_tables (AggState *aggstate)

static void	build_hash_table (AggState *aggstate, int setno, long nbuckets)

static void	hashagg_recompile_expressions (AggState *aggstate, bool minslot, bool nullcheck)

static void	hash_create_memory (AggState *aggstate)

static long	hash_choose_num_buckets (double hashentrysize, long ngroups, Size memory)

static int	hash_choose_num_partitions (double input_groups, double hashentrysize, int used_bits, int *log2_npartitions)

static void	initialize_hash_entry (AggState *aggstate, TupleHashTable hashtable, TupleHashEntry entry)

static void	lookup_hash_entries (AggState *aggstate)

static TupleTableSlot *	agg_retrieve_direct (AggState *aggstate)

static void	agg_fill_hash_table (AggState *aggstate)

static bool	agg_refill_hash_table (AggState *aggstate)

static TupleTableSlot *	agg_retrieve_hash_table (AggState *aggstate)

static TupleTableSlot *	agg_retrieve_hash_table_in_memory (AggState *aggstate)

static void	hash_agg_check_limits (AggState *aggstate)

static void	hash_agg_enter_spill_mode (AggState *aggstate)

static void	hash_agg_update_metrics (AggState *aggstate, bool from_tape, int npartitions)

static void	hashagg_finish_initial_spills (AggState *aggstate)

static void	hashagg_reset_spill_state (AggState *aggstate)

static HashAggBatch *	hashagg_batch_new (LogicalTape *input_tape, int setno, int64 input_tuples, double input_card, int used_bits)

static MinimalTuple	hashagg_batch_read (HashAggBatch batch, uint32 hashp)

static void	hashagg_spill_init (HashAggSpill spill, LogicalTapeSet tapeset, int used_bits, double input_groups, double hashentrysize)

static Size	hashagg_spill_tuple (AggState aggstate, HashAggSpill spill, TupleTableSlot *inputslot, uint32 hash)

static void	hashagg_spill_finish (AggState aggstate, HashAggSpill spill, int setno)

static Datum	GetAggInitVal (Datum textInitVal, Oid transtype)

static void	build_pertrans_for_aggref (AggStatePerTrans pertrans, AggState aggstate, EState estate, Aggref aggref, Oid transfn_oid, Oid aggtranstype, Oid aggserialfn, Oid aggdeserialfn, Datum initValue, bool initValueIsNull, Oid inputTypes, int numArguments)

static void	initialize_aggregate (AggState *aggstate, AggStatePerTrans pertrans, AggStatePerGroup pergroupstate)

static void	find_hash_columns (AggState *aggstate)

Size	hash_agg_entry_size (int numTrans, Size tupleWidth, Size transitionSpace)

void	hash_agg_set_limits (double hashentrysize, double input_groups, int used_bits, Size mem_limit, uint64 ngroups_limit, int *num_partitions)

static TupleTableSlot *	ExecAgg (PlanState *pstate)

AggState *	ExecInitAgg (Agg node, EState estate, int eflags)

void	ExecEndAgg (AggState *node)

void	ExecReScanAgg (AggState *node)

int	AggCheckCallContext (FunctionCallInfo fcinfo, MemoryContext *aggcontext)

Aggref *	AggGetAggref (FunctionCallInfo fcinfo)

MemoryContext	AggGetTempMemoryContext (FunctionCallInfo fcinfo)

bool	AggStateIsShared (FunctionCallInfo fcinfo)

void	AggRegisterCallback (FunctionCallInfo fcinfo, ExprContextCallbackFunction func, Datum arg)

void	ExecAggEstimate (AggState node, ParallelContext pcxt)

void	ExecAggInitializeDSM (AggState node, ParallelContext pcxt)

void	ExecAggInitializeWorker (AggState node, ParallelWorkerContext pwcxt)

void	ExecAggRetrieveInstrumentation (AggState *node)

Macro Definition Documentation

◆ CHUNKHDRSZ

#define CHUNKHDRSZ sizeof(MemoryChunk)

Definition at line 321 of file nodeAgg.c.

◆ HASHAGG_HLL_BIT_WIDTH

#define HASHAGG_HLL_BIT_WIDTH 5

Definition at line 316 of file nodeAgg.c.

◆ HASHAGG_MAX_PARTITIONS

#define HASHAGG_MAX_PARTITIONS 1024

Definition at line 299 of file nodeAgg.c.

◆ HASHAGG_MIN_PARTITIONS

#define HASHAGG_MIN_PARTITIONS 4

Definition at line 298 of file nodeAgg.c.

◆ HASHAGG_PARTITION_FACTOR

#define HASHAGG_PARTITION_FACTOR 1.50

Definition at line 297 of file nodeAgg.c.

◆ HASHAGG_READ_BUFFER_SIZE

#define HASHAGG_READ_BUFFER_SIZE BLCKSZ

Definition at line 307 of file nodeAgg.c.

◆ HASHAGG_WRITE_BUFFER_SIZE

#define HASHAGG_WRITE_BUFFER_SIZE BLCKSZ

Definition at line 308 of file nodeAgg.c.

Typedef Documentation

◆ FindColsContext

typedef struct FindColsContext FindColsContext

◆ HashAggBatch

typedef struct HashAggBatch HashAggBatch

◆ HashAggSpill

typedef struct HashAggSpill HashAggSpill

Function Documentation

◆ advance_aggregates()

static void advance_aggregates ( AggState * aggstate )

static

Definition at line 818 of file nodeAgg.c.

{
    ExecEvalExprNoReturnSwitchContext(aggstate->phase->evaltrans,
                                      aggstate->tmpcontext);
}

References AggStatePerPhaseData::evaltrans, ExecEvalExprNoReturnSwitchContext(), AggState::phase, and AggState::tmpcontext.

Referenced by agg_fill_hash_table(), agg_refill_hash_table(), and agg_retrieve_direct().

◆ advance_transition_function()

static void advance_transition_function	(	AggState *	aggstate,
		AggStatePerTrans	pertrans,
		AggStatePerGroup	pergroupstate
	)

static

Definition at line 708 of file nodeAgg.c.

{
    FunctionCallInfo fcinfo = pertrans->transfn_fcinfo;
    MemoryContext oldContext;
    Datum       newVal;
 
    if (pertrans->transfn.fn_strict)
    {
        /*
         * For a strict transfn, nothing happens when there's a NULL input; we
         * just keep the prior transValue.
         */
        int         numTransInputs = pertrans->numTransInputs;
        int         i;
 
        for (i = 1; i <= numTransInputs; i++)
        {
            if (fcinfo->args[i].isnull)
                return;
        }
        if (pergroupstate->noTransValue)
        {
            /*
             * transValue has not been initialized. This is the first non-NULL
             * input value. We use it as the initial value for transValue. (We
             * already checked that the agg's input type is binary-compatible
             * with its transtype, so straight copy here is OK.)
             *
             * We must copy the datum into aggcontext if it is pass-by-ref. We
             * do not need to pfree the old transValue, since it's NULL.
             */
            oldContext = MemoryContextSwitchTo(aggstate->curaggcontext->ecxt_per_tuple_memory);
            pergroupstate->transValue = datumCopy(fcinfo->args[1].value,
                                                  pertrans->transtypeByVal,
                                                  pertrans->transtypeLen);
            pergroupstate->transValueIsNull = false;
            pergroupstate->noTransValue = false;
            MemoryContextSwitchTo(oldContext);
            return;
        }
        if (pergroupstate->transValueIsNull)
        {
            /*
             * Don't call a strict function with NULL inputs.  Note it is
             * possible to get here despite the above tests, if the transfn is
             * strict *and* returned a NULL on a prior cycle. If that happens
             * we will propagate the NULL all the way to the end.
             */
            return;
        }
    }
 
    /* We run the transition functions in per-input-tuple memory context */
    oldContext = MemoryContextSwitchTo(aggstate->tmpcontext->ecxt_per_tuple_memory);
 
    /* set up aggstate->curpertrans for AggGetAggref() */
    aggstate->curpertrans = pertrans;
 
    /*
     * OK to call the transition function
     */
    fcinfo->args[0].value = pergroupstate->transValue;
    fcinfo->args[0].isnull = pergroupstate->transValueIsNull;
    fcinfo->isnull = false;     /* just in case transfn doesn't set it */
 
    newVal = FunctionCallInvoke(fcinfo);
 
    aggstate->curpertrans = NULL;
 
    /*
     * If pass-by-ref datatype, must copy the new value into aggcontext and
     * free the prior transValue.  But if transfn returned a pointer to its
     * first input, we don't need to do anything.
     *
     * It's safe to compare newVal with pergroup->transValue without regard
     * for either being NULL, because ExecAggCopyTransValue takes care to set
     * transValue to 0 when NULL. Otherwise we could end up accidentally not
     * reparenting, when the transValue has the same numerical value as
     * newValue, despite being NULL.  This is a somewhat hot path, making it
     * undesirable to instead solve this with another branch for the common
     * case of the transition function returning its (modified) input
     * argument.
     */
    if (!pertrans->transtypeByVal &&
        DatumGetPointer(newVal) != DatumGetPointer(pergroupstate->transValue))
        newVal = ExecAggCopyTransValue(aggstate, pertrans,
                                       newVal, fcinfo->isnull,
                                       pergroupstate->transValue,
                                       pergroupstate->transValueIsNull);
 
    pergroupstate->transValue = newVal;
    pergroupstate->transValueIsNull = fcinfo->isnull;
 
    MemoryContextSwitchTo(oldContext);
}

References FunctionCallInfoBaseData::args, AggState::curaggcontext, AggState::curpertrans, datumCopy(), DatumGetPointer(), ExprContext::ecxt_per_tuple_memory, ExecAggCopyTransValue(), FmgrInfo::fn_strict, FunctionCallInvoke, i, FunctionCallInfoBaseData::isnull, NullableDatum::isnull, MemoryContextSwitchTo(), AggStatePerGroupData::noTransValue, AggStatePerTransData::numTransInputs, AggState::tmpcontext, AggStatePerTransData::transfn, AggStatePerTransData::transfn_fcinfo, AggStatePerTransData::transtypeByVal, AggStatePerTransData::transtypeLen, AggStatePerGroupData::transValue, AggStatePerGroupData::transValueIsNull, and NullableDatum::value.

Referenced by process_ordered_aggregate_multi(), and process_ordered_aggregate_single().

◆ agg_fill_hash_table()

static void agg_fill_hash_table ( AggState * aggstate )

static

Definition at line 2626 of file nodeAgg.c.

{
    TupleTableSlot *outerslot;
    ExprContext *tmpcontext = aggstate->tmpcontext;
 
    /*
     * Process each outer-plan tuple, and then fetch the next one, until we
     * exhaust the outer plan.
     */
    for (;;)
    {
        outerslot = fetch_input_tuple(aggstate);
        if (TupIsNull(outerslot))
            break;
 
        /* set up for lookup_hash_entries and advance_aggregates */
        tmpcontext->ecxt_outertuple = outerslot;
 
        /* Find or build hashtable entries */
        lookup_hash_entries(aggstate);
 
        /* Advance the aggregates (or combine functions) */
        advance_aggregates(aggstate);
 
        /*
         * Reset per-input-tuple context after each tuple, but note that the
         * hash lookups do this too
         */
        ResetExprContext(aggstate->tmpcontext);
    }
 
    /* finalize spills, if any */
    hashagg_finish_initial_spills(aggstate);
 
    aggstate->table_filled = true;
    /* Initialize to walk the first hash table */
    select_current_set(aggstate, 0, true);
    ResetTupleHashIterator(aggstate->perhash[0].hashtable,
                           &aggstate->perhash[0].hashiter);
}

References advance_aggregates(), ExprContext::ecxt_outertuple, fetch_input_tuple(), hashagg_finish_initial_spills(), AggStatePerHashData::hashiter, AggStatePerHashData::hashtable, lookup_hash_entries(), AggState::perhash, ResetExprContext, ResetTupleHashIterator, select_current_set(), AggState::table_filled, AggState::tmpcontext, and TupIsNull.

Referenced by ExecAgg().

◆ agg_refill_hash_table()

static bool agg_refill_hash_table ( AggState * aggstate )

static

Definition at line 2680 of file nodeAgg.c.

{
    HashAggBatch *batch;
    AggStatePerHash perhash;
    HashAggSpill spill;
    LogicalTapeSet *tapeset = aggstate->hash_tapeset;
    bool        spill_initialized = false;
 
    if (aggstate->hash_batches == NIL)
        return false;
 
    /* hash_batches is a stack, with the top item at the end of the list */
    batch = llast(aggstate->hash_batches);
    aggstate->hash_batches = list_delete_last(aggstate->hash_batches);
 
    hash_agg_set_limits(aggstate->hashentrysize, batch->input_card,
                        batch->used_bits, &aggstate->hash_mem_limit,
                        &aggstate->hash_ngroups_limit, NULL);
 
    /*
     * Each batch only processes one grouping set; set the rest to NULL so
     * that advance_aggregates() knows to ignore them. We don't touch
     * pergroups for sorted grouping sets here, because they will be needed if
     * we rescan later. The expressions for sorted grouping sets will not be
     * evaluated after we recompile anyway.
     */
    MemSet(aggstate->hash_pergroup, 0,
           sizeof(AggStatePerGroup) * aggstate->num_hashes);
 
    /* free memory and reset hash tables */
    ReScanExprContext(aggstate->hashcontext);
    MemoryContextReset(aggstate->hash_tablecxt);
    for (int setno = 0; setno < aggstate->num_hashes; setno++)
        ResetTupleHashTable(aggstate->perhash[setno].hashtable);
 
    aggstate->hash_ngroups_current = 0;
 
    /*
     * In AGG_MIXED mode, hash aggregation happens in phase 1 and the output
     * happens in phase 0. So, we switch to phase 1 when processing a batch,
     * and back to phase 0 after the batch is done.
     */
    Assert(aggstate->current_phase == 0);
    if (aggstate->phase->aggstrategy == AGG_MIXED)
    {
        aggstate->current_phase = 1;
        aggstate->phase = &aggstate->phases[aggstate->current_phase];
    }
 
    select_current_set(aggstate, batch->setno, true);
 
    perhash = &aggstate->perhash[aggstate->current_set];
 
    /*
     * Spilled tuples are always read back as MinimalTuples, which may be
     * different from the outer plan, so recompile the aggregate expressions.
     *
     * We still need the NULL check, because we are only processing one
     * grouping set at a time and the rest will be NULL.
     */
    hashagg_recompile_expressions(aggstate, true, true);
 
    INJECTION_POINT("hash-aggregate-process-batch");
    for (;;)
    {
        TupleTableSlot *spillslot = aggstate->hash_spill_rslot;
        TupleTableSlot *hashslot = perhash->hashslot;
        TupleHashTable hashtable = perhash->hashtable;
        TupleHashEntry entry;
        MinimalTuple tuple;
        uint32      hash;
        bool        isnew = false;
        bool       *p_isnew = aggstate->hash_spill_mode ? NULL : &isnew;
 
        CHECK_FOR_INTERRUPTS();
 
        tuple = hashagg_batch_read(batch, &hash);
        if (tuple == NULL)
            break;
 
        ExecStoreMinimalTuple(tuple, spillslot, true);
        aggstate->tmpcontext->ecxt_outertuple = spillslot;
 
        prepare_hash_slot(perhash,
                          aggstate->tmpcontext->ecxt_outertuple,
                          hashslot);
        entry = LookupTupleHashEntryHash(hashtable, hashslot,
                                         p_isnew, hash);
 
        if (entry != NULL)
        {
            if (isnew)
                initialize_hash_entry(aggstate, hashtable, entry);
            aggstate->hash_pergroup[batch->setno] = TupleHashEntryGetAdditional(hashtable, entry);
            advance_aggregates(aggstate);
        }
        else
        {
            if (!spill_initialized)
            {
                /*
                 * Avoid initializing the spill until we actually need it so
                 * that we don't assign tapes that will never be used.
                 */
                spill_initialized = true;
                hashagg_spill_init(&spill, tapeset, batch->used_bits,
                                   batch->input_card, aggstate->hashentrysize);
            }
            /* no memory for a new group, spill */
            hashagg_spill_tuple(aggstate, &spill, spillslot, hash);
 
            aggstate->hash_pergroup[batch->setno] = NULL;
        }
 
        /*
         * Reset per-input-tuple context after each tuple, but note that the
         * hash lookups do this too
         */
        ResetExprContext(aggstate->tmpcontext);
    }
 
    LogicalTapeClose(batch->input_tape);
 
    /* change back to phase 0 */
    aggstate->current_phase = 0;
    aggstate->phase = &aggstate->phases[aggstate->current_phase];
 
    if (spill_initialized)
    {
        hashagg_spill_finish(aggstate, &spill, batch->setno);
        hash_agg_update_metrics(aggstate, true, spill.npartitions);
    }
    else
        hash_agg_update_metrics(aggstate, true, 0);
 
    aggstate->hash_spill_mode = false;
 
    /* prepare to walk the first hash table */
    select_current_set(aggstate, batch->setno, true);
    ResetTupleHashIterator(aggstate->perhash[batch->setno].hashtable,
                           &aggstate->perhash[batch->setno].hashiter);
 
    pfree(batch);
 
    return true;
}

References advance_aggregates(), AGG_MIXED, AggStatePerPhaseData::aggstrategy, Assert(), CHECK_FOR_INTERRUPTS, AggState::current_phase, AggState::current_set, ExprContext::ecxt_outertuple, ExecStoreMinimalTuple(), hash(), hash_agg_set_limits(), hash_agg_update_metrics(), AggState::hash_batches, AggState::hash_mem_limit, AggState::hash_ngroups_current, AggState::hash_ngroups_limit, AggState::hash_pergroup, AggState::hash_spill_mode, AggState::hash_spill_rslot, AggState::hash_tablecxt, AggState::hash_tapeset, hashagg_batch_read(), hashagg_recompile_expressions(), hashagg_spill_finish(), hashagg_spill_init(), hashagg_spill_tuple(), AggState::hashcontext, AggState::hashentrysize, AggStatePerHashData::hashiter, AggStatePerHashData::hashslot, AggStatePerHashData::hashtable, initialize_hash_entry(), INJECTION_POINT, HashAggBatch::input_card, HashAggBatch::input_tape, list_delete_last(), llast, LogicalTapeClose(), LookupTupleHashEntryHash(), MemoryContextReset(), MemSet, NIL, HashAggSpill::npartitions, AggState::num_hashes, AggState::perhash, pfree(), AggState::phase, AggState::phases, prepare_hash_slot(), ReScanExprContext(), ResetExprContext, ResetTupleHashIterator, ResetTupleHashTable(), select_current_set(), HashAggBatch::setno, AggState::tmpcontext, TupleHashEntryGetAdditional(), and HashAggBatch::used_bits.

Referenced by agg_retrieve_hash_table().

◆ agg_retrieve_direct()

static TupleTableSlot * agg_retrieve_direct ( AggState * aggstate )

static

Definition at line 2280 of file nodeAgg.c.

{
    Agg        *node = aggstate->phase->aggnode;
    ExprContext *econtext;
    ExprContext *tmpcontext;
    AggStatePerAgg peragg;
    AggStatePerGroup *pergroups;
    TupleTableSlot *outerslot;
    TupleTableSlot *firstSlot;
    TupleTableSlot *result;
    bool        hasGroupingSets = aggstate->phase->numsets > 0;
    int         numGroupingSets = Max(aggstate->phase->numsets, 1);
    int         currentSet;
    int         nextSetSize;
    int         numReset;
    int         i;
 
    /*
     * get state info from node
     *
     * econtext is the per-output-tuple expression context
     *
     * tmpcontext is the per-input-tuple expression context
     */
    econtext = aggstate->ss.ps.ps_ExprContext;
    tmpcontext = aggstate->tmpcontext;
 
    peragg = aggstate->peragg;
    pergroups = aggstate->pergroups;
    firstSlot = aggstate->ss.ss_ScanTupleSlot;
 
    /*
     * We loop retrieving groups until we find one matching
     * aggstate->ss.ps.qual
     *
     * For grouping sets, we have the invariant that aggstate->projected_set
     * is either -1 (initial call) or the index (starting from 0) in
     * gset_lengths for the group we just completed (either by projecting a
     * row or by discarding it in the qual).
     */
    while (!aggstate->agg_done)
    {
        /*
         * Clear the per-output-tuple context for each group, as well as
         * aggcontext (which contains any pass-by-ref transvalues of the old
         * group).  Some aggregate functions store working state in child
         * contexts; those now get reset automatically without us needing to
         * do anything special.
         *
         * We use ReScanExprContext not just ResetExprContext because we want
         * any registered shutdown callbacks to be called.  That allows
         * aggregate functions to ensure they've cleaned up any non-memory
         * resources.
         */
        ReScanExprContext(econtext);
 
        /*
         * Determine how many grouping sets need to be reset at this boundary.
         */
        if (aggstate->projected_set >= 0 &&
            aggstate->projected_set < numGroupingSets)
            numReset = aggstate->projected_set + 1;
        else
            numReset = numGroupingSets;
 
        /*
         * numReset can change on a phase boundary, but that's OK; we want to
         * reset the contexts used in _this_ phase, and later, after possibly
         * changing phase, initialize the right number of aggregates for the
         * _new_ phase.
         */
 
        for (i = 0; i < numReset; i++)
        {
            ReScanExprContext(aggstate->aggcontexts[i]);
        }
 
        /*
         * Check if input is complete and there are no more groups to project
         * in this phase; move to next phase or mark as done.
         */
        if (aggstate->input_done == true &&
            aggstate->projected_set >= (numGroupingSets - 1))
        {
            if (aggstate->current_phase < aggstate->numphases - 1)
            {
                initialize_phase(aggstate, aggstate->current_phase + 1);
                aggstate->input_done = false;
                aggstate->projected_set = -1;
                numGroupingSets = Max(aggstate->phase->numsets, 1);
                node = aggstate->phase->aggnode;
                numReset = numGroupingSets;
            }
            else if (aggstate->aggstrategy == AGG_MIXED)
            {
                /*
                 * Mixed mode; we've output all the grouped stuff and have
                 * full hashtables, so switch to outputting those.
                 */
                initialize_phase(aggstate, 0);
                aggstate->table_filled = true;
                ResetTupleHashIterator(aggstate->perhash[0].hashtable,
                                       &aggstate->perhash[0].hashiter);
                select_current_set(aggstate, 0, true);
                return agg_retrieve_hash_table(aggstate);
            }
            else
            {
                aggstate->agg_done = true;
                break;
            }
        }
 
        /*
         * Get the number of columns in the next grouping set after the last
         * projected one (if any). This is the number of columns to compare to
         * see if we reached the boundary of that set too.
         */
        if (aggstate->projected_set >= 0 &&
            aggstate->projected_set < (numGroupingSets - 1))
            nextSetSize = aggstate->phase->gset_lengths[aggstate->projected_set + 1];
        else
            nextSetSize = 0;
 
        /*----------
         * If a subgroup for the current grouping set is present, project it.
         *
         * We have a new group if:
         *  - we're out of input but haven't projected all grouping sets
         *    (checked above)
         * OR
         *    - we already projected a row that wasn't from the last grouping
         *      set
         *    AND
         *    - the next grouping set has at least one grouping column (since
         *      empty grouping sets project only once input is exhausted)
         *    AND
         *    - the previous and pending rows differ on the grouping columns
         *      of the next grouping set
         *----------
         */
        tmpcontext->ecxt_innertuple = econtext->ecxt_outertuple;
        if (aggstate->input_done ||
            (node->aggstrategy != AGG_PLAIN &&
             aggstate->projected_set != -1 &&
             aggstate->projected_set < (numGroupingSets - 1) &&
             nextSetSize > 0 &&
             !ExecQualAndReset(aggstate->phase->eqfunctions[nextSetSize - 1],
                               tmpcontext)))
        {
            aggstate->projected_set += 1;
 
            Assert(aggstate->projected_set < numGroupingSets);
            Assert(nextSetSize > 0 || aggstate->input_done);
        }
        else
        {
            /*
             * We no longer care what group we just projected, the next
             * projection will always be the first (or only) grouping set
             * (unless the input proves to be empty).
             */
            aggstate->projected_set = 0;
 
            /*
             * If we don't already have the first tuple of the new group,
             * fetch it from the outer plan.
             */
            if (aggstate->grp_firstTuple == NULL)
            {
                outerslot = fetch_input_tuple(aggstate);
                if (!TupIsNull(outerslot))
                {
                    /*
                     * Make a copy of the first input tuple; we will use this
                     * for comparisons (in group mode) and for projection.
                     */
                    aggstate->grp_firstTuple = ExecCopySlotHeapTuple(outerslot);
                }
                else
                {
                    /* outer plan produced no tuples at all */
                    if (hasGroupingSets)
                    {
                        /*
                         * If there was no input at all, we need to project
                         * rows only if there are grouping sets of size 0.
                         * Note that this implies that there can't be any
                         * references to ungrouped Vars, which would otherwise
                         * cause issues with the empty output slot.
                         *
                         * XXX: This is no longer true, we currently deal with
                         * this in finalize_aggregates().
                         */
                        aggstate->input_done = true;
 
                        while (aggstate->phase->gset_lengths[aggstate->projected_set] > 0)
                        {
                            aggstate->projected_set += 1;
                            if (aggstate->projected_set >= numGroupingSets)
                            {
                                /*
                                 * We can't set agg_done here because we might
                                 * have more phases to do, even though the
                                 * input is empty. So we need to restart the
                                 * whole outer loop.
                                 */
                                break;
                            }
                        }
 
                        if (aggstate->projected_set >= numGroupingSets)
                            continue;
                    }
                    else
                    {
                        aggstate->agg_done = true;
                        /* If we are grouping, we should produce no tuples too */
                        if (node->aggstrategy != AGG_PLAIN)
                            return NULL;
                    }
                }
            }
 
            /*
             * Initialize working state for a new input tuple group.
             */
            initialize_aggregates(aggstate, pergroups, numReset);
 
            if (aggstate->grp_firstTuple != NULL)
            {
                /*
                 * Store the copied first input tuple in the tuple table slot
                 * reserved for it.  The tuple will be deleted when it is
                 * cleared from the slot.
                 */
                ExecForceStoreHeapTuple(aggstate->grp_firstTuple,
                                        firstSlot, true);
                aggstate->grp_firstTuple = NULL;    /* don't keep two pointers */
 
                /* set up for first advance_aggregates call */
                tmpcontext->ecxt_outertuple = firstSlot;
 
                /*
                 * Process each outer-plan tuple, and then fetch the next one,
                 * until we exhaust the outer plan or cross a group boundary.
                 */
                for (;;)
                {
                    /*
                     * During phase 1 only of a mixed agg, we need to update
                     * hashtables as well in advance_aggregates.
                     */
                    if (aggstate->aggstrategy == AGG_MIXED &&
                        aggstate->current_phase == 1)
                    {
                        lookup_hash_entries(aggstate);
                    }
 
                    /* Advance the aggregates (or combine functions) */
                    advance_aggregates(aggstate);
 
                    /* Reset per-input-tuple context after each tuple */
                    ResetExprContext(tmpcontext);
 
                    outerslot = fetch_input_tuple(aggstate);
                    if (TupIsNull(outerslot))
                    {
                        /* no more outer-plan tuples available */
 
                        /* if we built hash tables, finalize any spills */
                        if (aggstate->aggstrategy == AGG_MIXED &&
                            aggstate->current_phase == 1)
                            hashagg_finish_initial_spills(aggstate);
 
                        if (hasGroupingSets)
                        {
                            aggstate->input_done = true;
                            break;
                        }
                        else
                        {
                            aggstate->agg_done = true;
                            break;
                        }
                    }
                    /* set up for next advance_aggregates call */
                    tmpcontext->ecxt_outertuple = outerslot;
 
                    /*
                     * If we are grouping, check whether we've crossed a group
                     * boundary.
                     */
                    if (node->aggstrategy != AGG_PLAIN && node->numCols > 0)
                    {
                        tmpcontext->ecxt_innertuple = firstSlot;
                        if (!ExecQual(aggstate->phase->eqfunctions[node->numCols - 1],
                                      tmpcontext))
                        {
                            aggstate->grp_firstTuple = ExecCopySlotHeapTuple(outerslot);
                            break;
                        }
                    }
                }
            }
 
            /*
             * Use the representative input tuple for any references to
             * non-aggregated input columns in aggregate direct args, the node
             * qual, and the tlist.  (If we are not grouping, and there are no
             * input rows at all, we will come here with an empty firstSlot
             * ... but if not grouping, there can't be any references to
             * non-aggregated input columns, so no problem.)
             */
            econtext->ecxt_outertuple = firstSlot;
        }
 
        Assert(aggstate->projected_set >= 0);
 
        currentSet = aggstate->projected_set;
 
        prepare_projection_slot(aggstate, econtext->ecxt_outertuple, currentSet);
 
        select_current_set(aggstate, currentSet, false);
 
        finalize_aggregates(aggstate,
                            peragg,
                            pergroups[currentSet]);
 
        /*
         * If there's no row to project right now, we must continue rather
         * than returning a null since there might be more groups.
         */
        result = project_aggregates(aggstate);
        if (result)
            return result;
    }
 
    /* No more groups */
    return NULL;
}

References advance_aggregates(), AggState::agg_done, AGG_MIXED, AGG_PLAIN, agg_retrieve_hash_table(), AggState::aggcontexts, AggStatePerPhaseData::aggnode, AggState::aggstrategy, Agg::aggstrategy, Assert(), AggState::current_phase, ExprContext::ecxt_innertuple, ExprContext::ecxt_outertuple, AggStatePerPhaseData::eqfunctions, ExecCopySlotHeapTuple(), ExecForceStoreHeapTuple(), ExecQual(), ExecQualAndReset(), fetch_input_tuple(), finalize_aggregates(), AggState::grp_firstTuple, AggStatePerPhaseData::gset_lengths, hashagg_finish_initial_spills(), AggStatePerHashData::hashiter, AggStatePerHashData::hashtable, i, initialize_aggregates(), initialize_phase(), AggState::input_done, lookup_hash_entries(), Max, Agg::numCols, AggState::numphases, AggStatePerPhaseData::numsets, AggState::peragg, AggState::pergroups, AggState::perhash, AggState::phase, prepare_projection_slot(), project_aggregates(), AggState::projected_set, ScanState::ps, PlanState::ps_ExprContext, ReScanExprContext(), ResetExprContext, ResetTupleHashIterator, select_current_set(), AggState::ss, ScanState::ss_ScanTupleSlot, AggState::table_filled, AggState::tmpcontext, and TupIsNull.

Referenced by ExecAgg().

◆ agg_retrieve_hash_table()

static TupleTableSlot * agg_retrieve_hash_table ( AggState * aggstate )

static

Definition at line 2835 of file nodeAgg.c.

{
    TupleTableSlot *result = NULL;
 
    while (result == NULL)
    {
        result = agg_retrieve_hash_table_in_memory(aggstate);
        if (result == NULL)
        {
            if (!agg_refill_hash_table(aggstate))
            {
                aggstate->agg_done = true;
                break;
            }
        }
    }
 
    return result;
}

References AggState::agg_done, agg_refill_hash_table(), and agg_retrieve_hash_table_in_memory().

Referenced by agg_retrieve_direct(), and ExecAgg().

◆ agg_retrieve_hash_table_in_memory()

static TupleTableSlot * agg_retrieve_hash_table_in_memory ( AggState * aggstate )

static

Definition at line 2860 of file nodeAgg.c.

{
    ExprContext *econtext;
    AggStatePerAgg peragg;
    AggStatePerGroup pergroup;
    TupleHashEntry entry;
    TupleTableSlot *firstSlot;
    TupleTableSlot *result;
    AggStatePerHash perhash;
 
    /*
     * get state info from node.
     *
     * econtext is the per-output-tuple expression context.
     */
    econtext = aggstate->ss.ps.ps_ExprContext;
    peragg = aggstate->peragg;
    firstSlot = aggstate->ss.ss_ScanTupleSlot;
 
    /*
     * Note that perhash (and therefore anything accessed through it) can
     * change inside the loop, as we change between grouping sets.
     */
    perhash = &aggstate->perhash[aggstate->current_set];
 
    /*
     * We loop retrieving groups until we find one satisfying
     * aggstate->ss.ps.qual
     */
    for (;;)
    {
        TupleTableSlot *hashslot = perhash->hashslot;
        TupleHashTable hashtable = perhash->hashtable;
        int         i;
 
        CHECK_FOR_INTERRUPTS();
 
        /*
         * Find the next entry in the hash table
         */
        entry = ScanTupleHashTable(hashtable, &perhash->hashiter);
        if (entry == NULL)
        {
            int         nextset = aggstate->current_set + 1;
 
            if (nextset < aggstate->num_hashes)
            {
                /*
                 * Switch to next grouping set, reinitialize, and restart the
                 * loop.
                 */
                select_current_set(aggstate, nextset, true);
 
                perhash = &aggstate->perhash[aggstate->current_set];
 
                ResetTupleHashIterator(hashtable, &perhash->hashiter);
 
                continue;
            }
            else
            {
                return NULL;
            }
        }
 
        /*
         * Clear the per-output-tuple context for each group
         *
         * We intentionally don't use ReScanExprContext here; if any aggs have
         * registered shutdown callbacks, they mustn't be called yet, since we
         * might not be done with that agg.
         */
        ResetExprContext(econtext);
 
        /*
         * Transform representative tuple back into one with the right
         * columns.
         */
        ExecStoreMinimalTuple(TupleHashEntryGetTuple(entry), hashslot, false);
        slot_getallattrs(hashslot);
 
        ExecClearTuple(firstSlot);
        memset(firstSlot->tts_isnull, true,
               firstSlot->tts_tupleDescriptor->natts * sizeof(bool));
 
        for (i = 0; i < perhash->numhashGrpCols; i++)
        {
            int         varNumber = perhash->hashGrpColIdxInput[i] - 1;
 
            firstSlot->tts_values[varNumber] = hashslot->tts_values[i];
            firstSlot->tts_isnull[varNumber] = hashslot->tts_isnull[i];
        }
        ExecStoreVirtualTuple(firstSlot);
 
        pergroup = (AggStatePerGroup) TupleHashEntryGetAdditional(hashtable, entry);
 
        /*
         * Use the representative input tuple for any references to
         * non-aggregated input columns in the qual and tlist.
         */
        econtext->ecxt_outertuple = firstSlot;
 
        prepare_projection_slot(aggstate,
                                econtext->ecxt_outertuple,
                                aggstate->current_set);
 
        finalize_aggregates(aggstate, peragg, pergroup);
 
        result = project_aggregates(aggstate);
        if (result)
            return result;
    }
 
    /* No more groups */
    return NULL;
}

References CHECK_FOR_INTERRUPTS, AggState::current_set, ExprContext::ecxt_outertuple, ExecClearTuple(), ExecStoreMinimalTuple(), ExecStoreVirtualTuple(), finalize_aggregates(), AggStatePerHashData::hashGrpColIdxInput, AggStatePerHashData::hashiter, AggStatePerHashData::hashslot, AggStatePerHashData::hashtable, i, TupleDescData::natts, AggStatePerHashData::numhashGrpCols, AggState::peragg, AggState::perhash, prepare_projection_slot(), project_aggregates(), ScanState::ps, PlanState::ps_ExprContext, ResetExprContext, ResetTupleHashIterator, ScanTupleHashTable, select_current_set(), slot_getallattrs(), AggState::ss, ScanState::ss_ScanTupleSlot, TupleTableSlot::tts_isnull, TupleTableSlot::tts_tupleDescriptor, TupleTableSlot::tts_values, TupleHashEntryGetAdditional(), and TupleHashEntryGetTuple().

Referenced by agg_retrieve_hash_table().

◆ AggCheckCallContext()

int AggCheckCallContext	(	FunctionCallInfo	fcinfo,
		MemoryContext *	aggcontext
	)

Definition at line 4614 of file nodeAgg.c.

{
    if (fcinfo->context && IsA(fcinfo->context, AggState))
    {
        if (aggcontext)
        {
            AggState   *aggstate = ((AggState *) fcinfo->context);
            ExprContext *cxt = aggstate->curaggcontext;
 
            *aggcontext = cxt->ecxt_per_tuple_memory;
        }
        return AGG_CONTEXT_AGGREGATE;
    }
    if (fcinfo->context && IsA(fcinfo->context, WindowAggState))
    {
        if (aggcontext)
            *aggcontext = ((WindowAggState *) fcinfo->context)->curaggcontext;
        return AGG_CONTEXT_WINDOW;
    }
 
    /* this is just to prevent "uninitialized variable" warnings */
    if (aggcontext)
        *aggcontext = NULL;
    return 0;
}

References AGG_CONTEXT_AGGREGATE, AGG_CONTEXT_WINDOW, FunctionCallInfoBaseData::context, AggState::curaggcontext, ExprContext::ecxt_per_tuple_memory, and IsA.

Referenced by array_agg_array_combine(), array_agg_array_deserialize(), array_agg_array_finalfn(), array_agg_array_serialize(), array_agg_array_transfn(), array_agg_combine(), array_agg_deserialize(), array_agg_finalfn(), array_agg_serialize(), array_agg_transfn(), bytea_string_agg_finalfn(), fetch_array_arg_replace_nulls(), float4_accum(), float8_accum(), float8_combine(), float8_regr_accum(), float8_regr_combine(), hypothetical_dense_rank_final(), hypothetical_rank_common(), int2_avg_accum(), int2_avg_accum_inv(), int2_sum(), int4_avg_accum(), int4_avg_accum_inv(), int4_avg_combine(), int4_sum(), int8_avg_combine(), int8_avg_deserialize(), int8_avg_serialize(), int8dec(), int8inc(), interval_avg_deserialize(), interval_avg_serialize(), json_agg_finalfn(), json_agg_transfn_worker(), json_object_agg_finalfn(), json_object_agg_transfn_worker(), jsonb_agg_finalfn(), jsonb_agg_transfn_worker(), jsonb_object_agg_finalfn(), jsonb_object_agg_transfn_worker(), makeBoolAggState(), makeIntervalAggState(), makeNumericAggState(), makeStringAggState(), mode_final(), multirange_agg_transfn(), multirange_intersect_agg_transfn(), numeric_avg_combine(), numeric_avg_deserialize(), numeric_avg_serialize(), numeric_combine(), numeric_deserialize(), numeric_poly_combine(), numeric_poly_deserialize(), numeric_poly_serialize(), numeric_serialize(), ordered_set_startup(), percentile_cont_final_common(), percentile_cont_multi_final_common(), percentile_disc_final(), percentile_disc_multi_final(), range_agg_finalfn(), range_agg_transfn(), range_intersect_agg_transfn(), string_agg_combine(), string_agg_deserialize(), string_agg_finalfn(), and string_agg_serialize().

◆ AggGetAggref()

Aggref * AggGetAggref ( FunctionCallInfo fcinfo )

Definition at line 4658 of file nodeAgg.c.

{
    if (fcinfo->context && IsA(fcinfo->context, AggState))
    {
        AggState   *aggstate = (AggState *) fcinfo->context;
        AggStatePerAgg curperagg;
        AggStatePerTrans curpertrans;
 
        /* check curperagg (valid when in a final function) */
        curperagg = aggstate->curperagg;
 
        if (curperagg)
            return curperagg->aggref;
 
        /* check curpertrans (valid when in a transition function) */
        curpertrans = aggstate->curpertrans;
 
        if (curpertrans)
            return curpertrans->aggref;
    }
    return NULL;
}

References AggStatePerTransData::aggref, FunctionCallInfoBaseData::context, AggState::curperagg, AggState::curpertrans, if(), and IsA.

Referenced by ordered_set_startup().

◆ AggGetTempMemoryContext()

MemoryContext AggGetTempMemoryContext ( FunctionCallInfo fcinfo )

Definition at line 4692 of file nodeAgg.c.

{
    if (fcinfo->context && IsA(fcinfo->context, AggState))
    {
        AggState   *aggstate = (AggState *) fcinfo->context;
 
        return aggstate->tmpcontext->ecxt_per_tuple_memory;
    }
    return NULL;
}

References FunctionCallInfoBaseData::context, ExprContext::ecxt_per_tuple_memory, IsA, and AggState::tmpcontext.

◆ AggRegisterCallback()

void AggRegisterCallback	(	FunctionCallInfo	fcinfo,
		ExprContextCallbackFunction	func,
		Datum	arg
	)

Definition at line 4757 of file nodeAgg.c.

{
    if (fcinfo->context && IsA(fcinfo->context, AggState))
    {
        AggState   *aggstate = (AggState *) fcinfo->context;
        ExprContext *cxt = aggstate->curaggcontext;
 
        RegisterExprContextCallback(cxt, func, arg);
 
        return;
    }
    elog(ERROR, "aggregate function cannot register a callback in this context");
}

References arg, FunctionCallInfoBaseData::context, AggState::curaggcontext, elog, ERROR, IsA, and RegisterExprContextCallback().

Referenced by ordered_set_startup().

◆ AggStateIsShared()

bool AggStateIsShared ( FunctionCallInfo fcinfo )

Definition at line 4718 of file nodeAgg.c.

{
    if (fcinfo->context && IsA(fcinfo->context, AggState))
    {
        AggState   *aggstate = (AggState *) fcinfo->context;
        AggStatePerAgg curperagg;
        AggStatePerTrans curpertrans;
 
        /* check curperagg (valid when in a final function) */
        curperagg = aggstate->curperagg;
 
        if (curperagg)
            return aggstate->pertrans[curperagg->transno].aggshared;
 
        /* check curpertrans (valid when in a transition function) */
        curpertrans = aggstate->curpertrans;
 
        if (curpertrans)
            return curpertrans->aggshared;
    }
    return true;
}

References AggStatePerTransData::aggshared, FunctionCallInfoBaseData::context, AggState::curperagg, AggState::curpertrans, if(), IsA, and AggState::pertrans.

Referenced by ordered_set_startup().

◆ build_hash_table()

static void build_hash_table	(	AggState *	aggstate,
		int	setno,
		long	nbuckets
	)

static

Definition at line 1509 of file nodeAgg.c.

{
    AggStatePerHash perhash = &aggstate->perhash[setno];
    MemoryContext metacxt = aggstate->hash_metacxt;
    MemoryContext tablecxt = aggstate->hash_tablecxt;
    MemoryContext tmpcxt = aggstate->tmpcontext->ecxt_per_tuple_memory;
    Size        additionalsize;
 
    Assert(aggstate->aggstrategy == AGG_HASHED ||
           aggstate->aggstrategy == AGG_MIXED);
 
    /*
     * Used to make sure initial hash table allocation does not exceed
     * hash_mem. Note that the estimate does not include space for
     * pass-by-reference transition data values, nor for the representative
     * tuple of each group.
     */
    additionalsize = aggstate->numtrans * sizeof(AggStatePerGroupData);
 
    perhash->hashtable = BuildTupleHashTable(&aggstate->ss.ps,
                                             perhash->hashslot->tts_tupleDescriptor,
                                             perhash->hashslot->tts_ops,
                                             perhash->numCols,
                                             perhash->hashGrpColIdxHash,
                                             perhash->eqfuncoids,
                                             perhash->hashfunctions,
                                             perhash->aggnode->grpCollations,
                                             nbuckets,
                                             additionalsize,
                                             metacxt,
                                             tablecxt,
                                             tmpcxt,
                                             DO_AGGSPLIT_SKIPFINAL(aggstate->aggsplit));
}

References AGG_HASHED, AGG_MIXED, AggStatePerHashData::aggnode, AggState::aggsplit, AggState::aggstrategy, Assert(), BuildTupleHashTable(), DO_AGGSPLIT_SKIPFINAL, ExprContext::ecxt_per_tuple_memory, AggStatePerHashData::eqfuncoids, AggState::hash_metacxt, AggState::hash_tablecxt, AggStatePerHashData::hashfunctions, AggStatePerHashData::hashGrpColIdxHash, AggStatePerHashData::hashslot, AggStatePerHashData::hashtable, AggStatePerHashData::numCols, AggState::numtrans, AggState::perhash, ScanState::ps, AggState::ss, AggState::tmpcontext, TupleTableSlot::tts_ops, and TupleTableSlot::tts_tupleDescriptor.

Referenced by build_hash_tables().

◆ build_hash_tables()

static void build_hash_tables ( AggState * aggstate )

static

Definition at line 1466 of file nodeAgg.c.

{
    int         setno;
 
    for (setno = 0; setno < aggstate->num_hashes; ++setno)
    {
        AggStatePerHash perhash = &aggstate->perhash[setno];
        long        nbuckets;
        Size        memory;
 
        if (perhash->hashtable != NULL)
        {
            ResetTupleHashTable(perhash->hashtable);
            continue;
        }
 
        Assert(perhash->aggnode->numGroups > 0);
 
        memory = aggstate->hash_mem_limit / aggstate->num_hashes;
 
        /* choose reasonable number of buckets per hashtable */
        nbuckets = hash_choose_num_buckets(aggstate->hashentrysize,
                                           perhash->aggnode->numGroups,
                                           memory);
 
#ifdef USE_INJECTION_POINTS
        if (IS_INJECTION_POINT_ATTACHED("hash-aggregate-oversize-table"))
        {
            nbuckets = memory / TupleHashEntrySize();
            INJECTION_POINT_CACHED("hash-aggregate-oversize-table");
        }
#endif
 
        build_hash_table(aggstate, setno, nbuckets);
    }
 
    aggstate->hash_ngroups_current = 0;
}

References AggStatePerHashData::aggnode, Assert(), build_hash_table(), hash_choose_num_buckets(), AggState::hash_mem_limit, AggState::hash_ngroups_current, AggState::hashentrysize, AggStatePerHashData::hashtable, INJECTION_POINT_CACHED, IS_INJECTION_POINT_ATTACHED, AggState::num_hashes, Agg::numGroups, AggState::perhash, ResetTupleHashTable(), and TupleHashEntrySize().

Referenced by ExecInitAgg(), and ExecReScanAgg().

◆ build_pertrans_for_aggref()

static void build_pertrans_for_aggref	(	AggStatePerTrans	pertrans,
		AggState *	aggstate,
		EState *	estate,
		Aggref *	aggref,
		Oid	transfn_oid,
		Oid	aggtranstype,
		Oid	aggserialfn,
		Oid	aggdeserialfn,
		Datum	initValue,
		bool	initValueIsNull,
		Oid *	inputTypes,
		int	numArguments
	)

static

Definition at line 4134 of file nodeAgg.c.

{
    int         numGroupingSets = Max(aggstate->maxsets, 1);
    Expr       *transfnexpr;
    int         numTransArgs;
    Expr       *serialfnexpr = NULL;
    Expr       *deserialfnexpr = NULL;
    ListCell   *lc;
    int         numInputs;
    int         numDirectArgs;
    List       *sortlist;
    int         numSortCols;
    int         numDistinctCols;
    int         i;
 
    /* Begin filling in the pertrans data */
    pertrans->aggref = aggref;
    pertrans->aggshared = false;
    pertrans->aggCollation = aggref->inputcollid;
    pertrans->transfn_oid = transfn_oid;
    pertrans->serialfn_oid = aggserialfn;
    pertrans->deserialfn_oid = aggdeserialfn;
    pertrans->initValue = initValue;
    pertrans->initValueIsNull = initValueIsNull;
 
    /* Count the "direct" arguments, if any */
    numDirectArgs = list_length(aggref->aggdirectargs);
 
    /* Count the number of aggregated input columns */
    pertrans->numInputs = numInputs = list_length(aggref->args);
 
    pertrans->aggtranstype = aggtranstype;
 
    /* account for the current transition state */
    numTransArgs = pertrans->numTransInputs + 1;
 
    /*
     * Set up infrastructure for calling the transfn.  Note that invtransfn is
     * not needed here.
     */
    build_aggregate_transfn_expr(inputTypes,
                                 numArguments,
                                 numDirectArgs,
                                 aggref->aggvariadic,
                                 aggtranstype,
                                 aggref->inputcollid,
                                 transfn_oid,
                                 InvalidOid,
                                 &transfnexpr,
                                 NULL);
 
    fmgr_info(transfn_oid, &pertrans->transfn);
    fmgr_info_set_expr((Node *) transfnexpr, &pertrans->transfn);
 
    pertrans->transfn_fcinfo =
        (FunctionCallInfo) palloc(SizeForFunctionCallInfo(numTransArgs));
    InitFunctionCallInfoData(*pertrans->transfn_fcinfo,
                             &pertrans->transfn,
                             numTransArgs,
                             pertrans->aggCollation,
                             (Node *) aggstate, NULL);
 
    /* get info about the state value's datatype */
    get_typlenbyval(aggtranstype,
                    &pertrans->transtypeLen,
                    &pertrans->transtypeByVal);
 
    if (OidIsValid(aggserialfn))
    {
        build_aggregate_serialfn_expr(aggserialfn,
                                      &serialfnexpr);
        fmgr_info(aggserialfn, &pertrans->serialfn);
        fmgr_info_set_expr((Node *) serialfnexpr, &pertrans->serialfn);
 
        pertrans->serialfn_fcinfo =
            (FunctionCallInfo) palloc(SizeForFunctionCallInfo(1));
        InitFunctionCallInfoData(*pertrans->serialfn_fcinfo,
                                 &pertrans->serialfn,
                                 1,
                                 InvalidOid,
                                 (Node *) aggstate, NULL);
    }
 
    if (OidIsValid(aggdeserialfn))
    {
        build_aggregate_deserialfn_expr(aggdeserialfn,
                                        &deserialfnexpr);
        fmgr_info(aggdeserialfn, &pertrans->deserialfn);
        fmgr_info_set_expr((Node *) deserialfnexpr, &pertrans->deserialfn);
 
        pertrans->deserialfn_fcinfo =
            (FunctionCallInfo) palloc(SizeForFunctionCallInfo(2));
        InitFunctionCallInfoData(*pertrans->deserialfn_fcinfo,
                                 &pertrans->deserialfn,
                                 2,
                                 InvalidOid,
                                 (Node *) aggstate, NULL);
    }
 
    /*
     * If we're doing either DISTINCT or ORDER BY for a plain agg, then we
     * have a list of SortGroupClause nodes; fish out the data in them and
     * stick them into arrays.  We ignore ORDER BY for an ordered-set agg,
     * however; the agg's transfn and finalfn are responsible for that.
     *
     * When the planner has set the aggpresorted flag, the input to the
     * aggregate is already correctly sorted.  For ORDER BY aggregates we can
     * simply treat these as normal aggregates.  For presorted DISTINCT
     * aggregates an extra step must be added to remove duplicate consecutive
     * inputs.
     *
     * Note that by construction, if there is a DISTINCT clause then the ORDER
     * BY clause is a prefix of it (see transformDistinctClause).
     */
    if (AGGKIND_IS_ORDERED_SET(aggref->aggkind))
    {
        sortlist = NIL;
        numSortCols = numDistinctCols = 0;
        pertrans->aggsortrequired = false;
    }
    else if (aggref->aggpresorted && aggref->aggdistinct == NIL)
    {
        sortlist = NIL;
        numSortCols = numDistinctCols = 0;
        pertrans->aggsortrequired = false;
    }
    else if (aggref->aggdistinct)
    {
        sortlist = aggref->aggdistinct;
        numSortCols = numDistinctCols = list_length(sortlist);
        Assert(numSortCols >= list_length(aggref->aggorder));
        pertrans->aggsortrequired = !aggref->aggpresorted;
    }
    else
    {
        sortlist = aggref->aggorder;
        numSortCols = list_length(sortlist);
        numDistinctCols = 0;
        pertrans->aggsortrequired = (numSortCols > 0);
    }
 
    pertrans->numSortCols = numSortCols;
    pertrans->numDistinctCols = numDistinctCols;
 
    /*
     * If we have either sorting or filtering to do, create a tupledesc and
     * slot corresponding to the aggregated inputs (including sort
     * expressions) of the agg.
     */
    if (numSortCols > 0 || aggref->aggfilter)
    {
        pertrans->sortdesc = ExecTypeFromTL(aggref->args);
        pertrans->sortslot =
            ExecInitExtraTupleSlot(estate, pertrans->sortdesc,
                                   &TTSOpsMinimalTuple);
    }
 
    if (numSortCols > 0)
    {
        /*
         * We don't implement DISTINCT or ORDER BY aggs in the HASHED case
         * (yet)
         */
        Assert(aggstate->aggstrategy != AGG_HASHED && aggstate->aggstrategy != AGG_MIXED);
 
        /* ORDER BY aggregates are not supported with partial aggregation */
        Assert(!DO_AGGSPLIT_COMBINE(aggstate->aggsplit));
 
        /* If we have only one input, we need its len/byval info. */
        if (numInputs == 1)
        {
            get_typlenbyval(inputTypes[numDirectArgs],
                            &pertrans->inputtypeLen,
                            &pertrans->inputtypeByVal);
        }
        else if (numDistinctCols > 0)
        {
            /* we will need an extra slot to store prior values */
            pertrans->uniqslot =
                ExecInitExtraTupleSlot(estate, pertrans->sortdesc,
                                       &TTSOpsMinimalTuple);
        }
 
        /* Extract the sort information for use later */
        pertrans->sortColIdx =
            (AttrNumber *) palloc(numSortCols * sizeof(AttrNumber));
        pertrans->sortOperators =
            (Oid *) palloc(numSortCols * sizeof(Oid));
        pertrans->sortCollations =
            (Oid *) palloc(numSortCols * sizeof(Oid));
        pertrans->sortNullsFirst =
            (bool *) palloc(numSortCols * sizeof(bool));
 
        i = 0;
        foreach(lc, sortlist)
        {
            SortGroupClause *sortcl = (SortGroupClause *) lfirst(lc);
            TargetEntry *tle = get_sortgroupclause_tle(sortcl, aggref->args);
 
            /* the parser should have made sure of this */
            Assert(OidIsValid(sortcl->sortop));
 
            pertrans->sortColIdx[i] = tle->resno;
            pertrans->sortOperators[i] = sortcl->sortop;
            pertrans->sortCollations[i] = exprCollation((Node *) tle->expr);
            pertrans->sortNullsFirst[i] = sortcl->nulls_first;
            i++;
        }
        Assert(i == numSortCols);
    }
 
    if (aggref->aggdistinct)
    {
        Oid        *ops;
 
        Assert(numArguments > 0);
        Assert(list_length(aggref->aggdistinct) == numDistinctCols);
 
        ops = palloc(numDistinctCols * sizeof(Oid));
 
        i = 0;
        foreach(lc, aggref->aggdistinct)
            ops[i++] = ((SortGroupClause *) lfirst(lc))->eqop;
 
        /* lookup / build the necessary comparators */
        if (numDistinctCols == 1)
            fmgr_info(get_opcode(ops[0]), &pertrans->equalfnOne);
        else
            pertrans->equalfnMulti =
                execTuplesMatchPrepare(pertrans->sortdesc,
                                       numDistinctCols,
                                       pertrans->sortColIdx,
                                       ops,
                                       pertrans->sortCollations,
                                       &aggstate->ss.ps);
        pfree(ops);
    }
 
    pertrans->sortstates = (Tuplesortstate **)
        palloc0(sizeof(Tuplesortstate *) * numGroupingSets);
}

Referenced by ExecInitAgg().

◆ ExecAgg()

static TupleTableSlot * ExecAgg ( PlanState * pstate )

static

Definition at line 2244 of file nodeAgg.c.

{
    AggState   *node = castNode(AggState, pstate);
    TupleTableSlot *result = NULL;
 
    CHECK_FOR_INTERRUPTS();
 
    if (!node->agg_done)
    {
        /* Dispatch based on strategy */
        switch (node->phase->aggstrategy)
        {
            case AGG_HASHED:
                if (!node->table_filled)
                    agg_fill_hash_table(node);
                /* FALLTHROUGH */
            case AGG_MIXED:
                result = agg_retrieve_hash_table(node);
                break;
            case AGG_PLAIN:
            case AGG_SORTED:
                result = agg_retrieve_direct(node);
                break;
        }
 
        if (!TupIsNull(result))
            return result;
    }
 
    return NULL;
}

References AggState::agg_done, agg_fill_hash_table(), AGG_HASHED, AGG_MIXED, AGG_PLAIN, agg_retrieve_direct(), agg_retrieve_hash_table(), AGG_SORTED, AggStatePerPhaseData::aggstrategy, castNode, CHECK_FOR_INTERRUPTS, AggState::phase, AggState::table_filled, and TupIsNull.

Referenced by ExecInitAgg().

◆ ExecAggEstimate()

void ExecAggEstimate	(	AggState *	node,
		ParallelContext *	pcxt
	)

Definition at line 4786 of file nodeAgg.c.

{
    Size        size;
 
    /* don't need this if not instrumenting or no workers */
    if (!node->ss.ps.instrument || pcxt->nworkers == 0)
        return;
 
    size = mul_size(pcxt->nworkers, sizeof(AggregateInstrumentation));
    size = add_size(size, offsetof(SharedAggInfo, sinstrument));
    shm_toc_estimate_chunk(&pcxt->estimator, size);
    shm_toc_estimate_keys(&pcxt->estimator, 1);
}

References add_size(), ParallelContext::estimator, PlanState::instrument, mul_size(), ParallelContext::nworkers, ScanState::ps, shm_toc_estimate_chunk, shm_toc_estimate_keys, and AggState::ss.

Referenced by ExecParallelEstimate().

◆ ExecAggInitializeDSM()

void ExecAggInitializeDSM	(	AggState *	node,
		ParallelContext *	pcxt
	)

Definition at line 4807 of file nodeAgg.c.

{
    Size        size;
 
    /* don't need this if not instrumenting or no workers */
    if (!node->ss.ps.instrument || pcxt->nworkers == 0)
        return;
 
    size = offsetof(SharedAggInfo, sinstrument)
        + pcxt->nworkers * sizeof(AggregateInstrumentation);
    node->shared_info = shm_toc_allocate(pcxt->toc, size);
    /* ensure any unfilled slots will contain zeroes */
    memset(node->shared_info, 0, size);
    node->shared_info->num_workers = pcxt->nworkers;
    shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id,
                   node->shared_info);
}

References PlanState::instrument, SharedAggInfo::num_workers, ParallelContext::nworkers, PlanState::plan, Plan::plan_node_id, ScanState::ps, AggState::shared_info, shm_toc_allocate(), shm_toc_insert(), AggState::ss, and ParallelContext::toc.

Referenced by ExecParallelInitializeDSM().

◆ ExecAggInitializeWorker()

void ExecAggInitializeWorker	(	AggState *	node,
		ParallelWorkerContext *	pwcxt
	)

Definition at line 4832 of file nodeAgg.c.

{
    node->shared_info =
        shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, true);
}

References PlanState::plan, Plan::plan_node_id, ScanState::ps, AggState::shared_info, shm_toc_lookup(), AggState::ss, and ParallelWorkerContext::toc.

Referenced by ExecParallelInitializeWorker().

◆ ExecAggRetrieveInstrumentation()

void ExecAggRetrieveInstrumentation ( AggState * node )

Definition at line 4845 of file nodeAgg.c.

{
    Size        size;
    SharedAggInfo *si;
 
    if (node->shared_info == NULL)
        return;
 
    size = offsetof(SharedAggInfo, sinstrument)
        + node->shared_info->num_workers * sizeof(AggregateInstrumentation);
    si = palloc(size);
    memcpy(si, node->shared_info, size);
    node->shared_info = si;
}

References SharedAggInfo::num_workers, palloc(), and AggState::shared_info.

Referenced by ExecParallelRetrieveInstrumentation().

◆ ExecEndAgg()

void ExecEndAgg ( AggState * node )

Definition at line 4400 of file nodeAgg.c.

{
    PlanState  *outerPlan;
    int         transno;
    int         numGroupingSets = Max(node->maxsets, 1);
    int         setno;
 
    /*
     * When ending a parallel worker, copy the statistics gathered by the
     * worker back into shared memory so that it can be picked up by the main
     * process to report in EXPLAIN ANALYZE.
     */
    if (node->shared_info && IsParallelWorker())
    {
        AggregateInstrumentation *si;
 
        Assert(ParallelWorkerNumber <= node->shared_info->num_workers);
        si = &node->shared_info->sinstrument[ParallelWorkerNumber];
        si->hash_batches_used = node->hash_batches_used;
        si->hash_disk_used = node->hash_disk_used;
        si->hash_mem_peak = node->hash_mem_peak;
    }
 
    /* Make sure we have closed any open tuplesorts */
 
    if (node->sort_in)
        tuplesort_end(node->sort_in);
    if (node->sort_out)
        tuplesort_end(node->sort_out);
 
    hashagg_reset_spill_state(node);
 
    if (node->hash_metacxt != NULL)
    {
        MemoryContextDelete(node->hash_metacxt);
        node->hash_metacxt = NULL;
    }
    if (node->hash_tablecxt != NULL)
    {
        MemoryContextDelete(node->hash_tablecxt);
        node->hash_tablecxt = NULL;
    }
 
 
    for (transno = 0; transno < node->numtrans; transno++)
    {
        AggStatePerTrans pertrans = &node->pertrans[transno];
 
        for (setno = 0; setno < numGroupingSets; setno++)
        {
            if (pertrans->sortstates[setno])
                tuplesort_end(pertrans->sortstates[setno]);
        }
    }
 
    /* And ensure any agg shutdown callbacks have been called */
    for (setno = 0; setno < numGroupingSets; setno++)
        ReScanExprContext(node->aggcontexts[setno]);
    if (node->hashcontext)
        ReScanExprContext(node->hashcontext);
 
    outerPlan = outerPlanState(node);
    ExecEndNode(outerPlan);
}

References AggState::aggcontexts, Assert(), ExecEndNode(), AggregateInstrumentation::hash_batches_used, AggState::hash_batches_used, AggregateInstrumentation::hash_disk_used, AggState::hash_disk_used, AggregateInstrumentation::hash_mem_peak, AggState::hash_mem_peak, AggState::hash_metacxt, AggState::hash_tablecxt, hashagg_reset_spill_state(), AggState::hashcontext, IsParallelWorker, Max, AggState::maxsets, MemoryContextDelete(), AggState::numtrans, outerPlan, outerPlanState, ParallelWorkerNumber, AggState::pertrans, ReScanExprContext(), AggState::shared_info, SharedAggInfo::sinstrument, AggState::sort_in, AggState::sort_out, AggStatePerTransData::sortstates, and tuplesort_end().

Referenced by ExecEndNode().

◆ ExecInitAgg()

AggState * ExecInitAgg	(	Agg *	node,
		EState *	estate,
		int	eflags
	)

Definition at line 3279 of file nodeAgg.c.

{
    AggState   *aggstate;
    AggStatePerAgg peraggs;
    AggStatePerTrans pertransstates;
    AggStatePerGroup *pergroups;
    Plan       *outerPlan;
    ExprContext *econtext;
    TupleDesc   scanDesc;
    int         max_aggno;
    int         max_transno;
    int         numaggrefs;
    int         numaggs;
    int         numtrans;
    int         phase;
    int         phaseidx;
    ListCell   *l;
    Bitmapset  *all_grouped_cols = NULL;
    int         numGroupingSets = 1;
    int         numPhases;
    int         numHashes;
    int         i = 0;
    int         j = 0;
    bool        use_hashing = (node->aggstrategy == AGG_HASHED ||
                               node->aggstrategy == AGG_MIXED);
 
    /* check for unsupported flags */
    Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
    /*
     * create state structure
     */
    aggstate = makeNode(AggState);
    aggstate->ss.ps.plan = (Plan *) node;
    aggstate->ss.ps.state = estate;
    aggstate->ss.ps.ExecProcNode = ExecAgg;
 
    aggstate->aggs = NIL;
    aggstate->numaggs = 0;
    aggstate->numtrans = 0;
    aggstate->aggstrategy = node->aggstrategy;
    aggstate->aggsplit = node->aggsplit;
    aggstate->maxsets = 0;
    aggstate->projected_set = -1;
    aggstate->current_set = 0;
    aggstate->peragg = NULL;
    aggstate->pertrans = NULL;
    aggstate->curperagg = NULL;
    aggstate->curpertrans = NULL;
    aggstate->input_done = false;
    aggstate->agg_done = false;
    aggstate->pergroups = NULL;
    aggstate->grp_firstTuple = NULL;
    aggstate->sort_in = NULL;
    aggstate->sort_out = NULL;
 
    /*
     * phases[0] always exists, but is dummy in sorted/plain mode
     */
    numPhases = (use_hashing ? 1 : 2);
    numHashes = (use_hashing ? 1 : 0);
 
    /*
     * Calculate the maximum number of grouping sets in any phase; this
     * determines the size of some allocations.  Also calculate the number of
     * phases, since all hashed/mixed nodes contribute to only a single phase.
     */
    if (node->groupingSets)
    {
        numGroupingSets = list_length(node->groupingSets);
 
        foreach(l, node->chain)
        {
            Agg        *agg = lfirst(l);
 
            numGroupingSets = Max(numGroupingSets,
                                  list_length(agg->groupingSets));
 
            /*
             * additional AGG_HASHED aggs become part of phase 0, but all
             * others add an extra phase.
             */
            if (agg->aggstrategy != AGG_HASHED)
                ++numPhases;
            else
                ++numHashes;
        }
    }
 
    aggstate->maxsets = numGroupingSets;
    aggstate->numphases = numPhases;
 
    aggstate->aggcontexts = (ExprContext **)
        palloc0(sizeof(ExprContext *) * numGroupingSets);
 
    /*
     * Create expression contexts.  We need three or more, one for
     * per-input-tuple processing, one for per-output-tuple processing, one
     * for all the hashtables, and one for each grouping set.  The per-tuple
     * memory context of the per-grouping-set ExprContexts (aggcontexts)
     * replaces the standalone memory context formerly used to hold transition
     * values.  We cheat a little by using ExecAssignExprContext() to build
     * all of them.
     *
     * NOTE: the details of what is stored in aggcontexts and what is stored
     * in the regular per-query memory context are driven by a simple
     * decision: we want to reset the aggcontext at group boundaries (if not
     * hashing) and in ExecReScanAgg to recover no-longer-wanted space.
     */
    ExecAssignExprContext(estate, &aggstate->ss.ps);
    aggstate->tmpcontext = aggstate->ss.ps.ps_ExprContext;
 
    for (i = 0; i < numGroupingSets; ++i)
    {
        ExecAssignExprContext(estate, &aggstate->ss.ps);
        aggstate->aggcontexts[i] = aggstate->ss.ps.ps_ExprContext;
    }
 
    if (use_hashing)
        hash_create_memory(aggstate);
 
    ExecAssignExprContext(estate, &aggstate->ss.ps);
 
    /*
     * Initialize child nodes.
     *
     * If we are doing a hashed aggregation then the child plan does not need
     * to handle REWIND efficiently; see ExecReScanAgg.
     */
    if (node->aggstrategy == AGG_HASHED)
        eflags &= ~EXEC_FLAG_REWIND;
    outerPlan = outerPlan(node);
    outerPlanState(aggstate) = ExecInitNode(outerPlan, estate, eflags);
 
    /*
     * initialize source tuple type.
     */
    aggstate->ss.ps.outerops =
        ExecGetResultSlotOps(outerPlanState(&aggstate->ss),
                             &aggstate->ss.ps.outeropsfixed);
    aggstate->ss.ps.outeropsset = true;
 
    ExecCreateScanSlotFromOuterPlan(estate, &aggstate->ss,
                                    aggstate->ss.ps.outerops);
    scanDesc = aggstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor;
 
    /*
     * If there are more than two phases (including a potential dummy phase
     * 0), input will be resorted using tuplesort. Need a slot for that.
     */
    if (numPhases > 2)
    {
        aggstate->sort_slot = ExecInitExtraTupleSlot(estate, scanDesc,
                                                     &TTSOpsMinimalTuple);
 
        /*
         * The output of the tuplesort, and the output from the outer child
         * might not use the same type of slot. In most cases the child will
         * be a Sort, and thus return a TTSOpsMinimalTuple type slot - but the
         * input can also be presorted due an index, in which case it could be
         * a different type of slot.
         *
         * XXX: For efficiency it would be good to instead/additionally
         * generate expressions with corresponding settings of outerops* for
         * the individual phases - deforming is often a bottleneck for
         * aggregations with lots of rows per group. If there's multiple
         * sorts, we know that all but the first use TTSOpsMinimalTuple (via
         * the nodeAgg.c internal tuplesort).
         */
        if (aggstate->ss.ps.outeropsfixed &&
            aggstate->ss.ps.outerops != &TTSOpsMinimalTuple)
            aggstate->ss.ps.outeropsfixed = false;
    }
 
    /*
     * Initialize result type, slot and projection.
     */
    ExecInitResultTupleSlotTL(&aggstate->ss.ps, &TTSOpsVirtual);
    ExecAssignProjectionInfo(&aggstate->ss.ps, NULL);
 
    /*
     * initialize child expressions
     *
     * We expect the parser to have checked that no aggs contain other agg
     * calls in their arguments (and just to be sure, we verify it again while
     * initializing the plan node).  This would make no sense under SQL
     * semantics, and it's forbidden by the spec.  Because it is true, we
     * don't need to worry about evaluating the aggs in any particular order.
     *
     * Note: execExpr.c finds Aggrefs for us, and adds them to aggstate->aggs.
     * Aggrefs in the qual are found here; Aggrefs in the targetlist are found
     * during ExecAssignProjectionInfo, above.
     */
    aggstate->ss.ps.qual =
        ExecInitQual(node->plan.qual, (PlanState *) aggstate);
 
    /*
     * We should now have found all Aggrefs in the targetlist and quals.
     */
    numaggrefs = list_length(aggstate->aggs);
    max_aggno = -1;
    max_transno = -1;
    foreach(l, aggstate->aggs)
    {
        Aggref     *aggref = (Aggref *) lfirst(l);
 
        max_aggno = Max(max_aggno, aggref->aggno);
        max_transno = Max(max_transno, aggref->aggtransno);
    }
    aggstate->numaggs = numaggs = max_aggno + 1;
    aggstate->numtrans = numtrans = max_transno + 1;
 
    /*
     * For each phase, prepare grouping set data and fmgr lookup data for
     * compare functions.  Accumulate all_grouped_cols in passing.
     */
    aggstate->phases = palloc0(numPhases * sizeof(AggStatePerPhaseData));
 
    aggstate->num_hashes = numHashes;
    if (numHashes)
    {
        aggstate->perhash = palloc0(sizeof(AggStatePerHashData) * numHashes);
        aggstate->phases[0].numsets = 0;
        aggstate->phases[0].gset_lengths = palloc(numHashes * sizeof(int));
        aggstate->phases[0].grouped_cols = palloc(numHashes * sizeof(Bitmapset *));
    }
 
    phase = 0;
    for (phaseidx = 0; phaseidx <= list_length(node->chain); ++phaseidx)
    {
        Agg        *aggnode;
        Sort       *sortnode;
 
        if (phaseidx > 0)
        {
            aggnode = list_nth_node(Agg, node->chain, phaseidx - 1);
            sortnode = castNode(Sort, outerPlan(aggnode));
        }
        else
        {
            aggnode = node;
            sortnode = NULL;
        }
 
        Assert(phase <= 1 || sortnode);
 
        if (aggnode->aggstrategy == AGG_HASHED
            || aggnode->aggstrategy == AGG_MIXED)
        {
            AggStatePerPhase phasedata = &aggstate->phases[0];
            AggStatePerHash perhash;
            Bitmapset  *cols = NULL;
 
            Assert(phase == 0);
            i = phasedata->numsets++;
            perhash = &aggstate->perhash[i];
 
            /* phase 0 always points to the "real" Agg in the hash case */
            phasedata->aggnode = node;
            phasedata->aggstrategy = node->aggstrategy;
 
            /* but the actual Agg node representing this hash is saved here */
            perhash->aggnode = aggnode;
 
            phasedata->gset_lengths[i] = perhash->numCols = aggnode->numCols;
 
            for (j = 0; j < aggnode->numCols; ++j)
                cols = bms_add_member(cols, aggnode->grpColIdx[j]);
 
            phasedata->grouped_cols[i] = cols;
 
            all_grouped_cols = bms_add_members(all_grouped_cols, cols);
            continue;
        }
        else
        {
            AggStatePerPhase phasedata = &aggstate->phases[++phase];
            int         num_sets;
 
            phasedata->numsets = num_sets = list_length(aggnode->groupingSets);
 
            if (num_sets)
            {
                phasedata->gset_lengths = palloc(num_sets * sizeof(int));
                phasedata->grouped_cols = palloc(num_sets * sizeof(Bitmapset *));
 
                i = 0;
                foreach(l, aggnode->groupingSets)
                {
                    int         current_length = list_length(lfirst(l));
                    Bitmapset  *cols = NULL;
 
                    /* planner forces this to be correct */
                    for (j = 0; j < current_length; ++j)
                        cols = bms_add_member(cols, aggnode->grpColIdx[j]);
 
                    phasedata->grouped_cols[i] = cols;
                    phasedata->gset_lengths[i] = current_length;
 
                    ++i;
                }
 
                all_grouped_cols = bms_add_members(all_grouped_cols,
                                                   phasedata->grouped_cols[0]);
            }
            else
            {
                Assert(phaseidx == 0);
 
                phasedata->gset_lengths = NULL;
                phasedata->grouped_cols = NULL;
            }
 
            /*
             * If we are grouping, precompute fmgr lookup data for inner loop.
             */
            if (aggnode->aggstrategy == AGG_SORTED)
            {
                /*
                 * Build a separate function for each subset of columns that
                 * need to be compared.
                 */
                phasedata->eqfunctions =
                    (ExprState **) palloc0(aggnode->numCols * sizeof(ExprState *));
 
                /* for each grouping set */
                for (int k = 0; k < phasedata->numsets; k++)
                {
                    int         length = phasedata->gset_lengths[k];
 
                    /* nothing to do for empty grouping set */
                    if (length == 0)
                        continue;
 
                    /* if we already had one of this length, it'll do */
                    if (phasedata->eqfunctions[length - 1] != NULL)
                        continue;
 
                    phasedata->eqfunctions[length - 1] =
                        execTuplesMatchPrepare(scanDesc,
                                               length,
                                               aggnode->grpColIdx,
                                               aggnode->grpOperators,
                                               aggnode->grpCollations,
                                               (PlanState *) aggstate);
                }
 
                /* and for all grouped columns, unless already computed */
                if (aggnode->numCols > 0 &&
                    phasedata->eqfunctions[aggnode->numCols - 1] == NULL)
                {
                    phasedata->eqfunctions[aggnode->numCols - 1] =
                        execTuplesMatchPrepare(scanDesc,
                                               aggnode->numCols,
                                               aggnode->grpColIdx,
                                               aggnode->grpOperators,
                                               aggnode->grpCollations,
                                               (PlanState *) aggstate);
                }
            }
 
            phasedata->aggnode = aggnode;
            phasedata->aggstrategy = aggnode->aggstrategy;
            phasedata->sortnode = sortnode;
        }
    }
 
    /*
     * Convert all_grouped_cols to a descending-order list.
     */
    i = -1;
    while ((i = bms_next_member(all_grouped_cols, i)) >= 0)
        aggstate->all_grouped_cols = lcons_int(i, aggstate->all_grouped_cols);
 
    /*
     * Set up aggregate-result storage in the output expr context, and also
     * allocate my private per-agg working storage
     */
    econtext = aggstate->ss.ps.ps_ExprContext;
    econtext->ecxt_aggvalues = (Datum *) palloc0(sizeof(Datum) * numaggs);
    econtext->ecxt_aggnulls = (bool *) palloc0(sizeof(bool) * numaggs);
 
    peraggs = (AggStatePerAgg) palloc0(sizeof(AggStatePerAggData) * numaggs);
    pertransstates = (AggStatePerTrans) palloc0(sizeof(AggStatePerTransData) * numtrans);
 
    aggstate->peragg = peraggs;
    aggstate->pertrans = pertransstates;
 
 
    aggstate->all_pergroups =
        (AggStatePerGroup *) palloc0(sizeof(AggStatePerGroup)
                                     * (numGroupingSets + numHashes));
    pergroups = aggstate->all_pergroups;
 
    if (node->aggstrategy != AGG_HASHED)
    {
        for (i = 0; i < numGroupingSets; i++)
        {
            pergroups[i] = (AggStatePerGroup) palloc0(sizeof(AggStatePerGroupData)
                                                      * numaggs);
        }
 
        aggstate->pergroups = pergroups;
        pergroups += numGroupingSets;
    }
 
    /*
     * Hashing can only appear in the initial phase.
     */
    if (use_hashing)
    {
        Plan       *outerplan = outerPlan(node);
        uint64      totalGroups = 0;
 
        aggstate->hash_spill_rslot = ExecInitExtraTupleSlot(estate, scanDesc,
                                                            &TTSOpsMinimalTuple);
        aggstate->hash_spill_wslot = ExecInitExtraTupleSlot(estate, scanDesc,
                                                            &TTSOpsVirtual);
 
        /* this is an array of pointers, not structures */
        aggstate->hash_pergroup = pergroups;
 
        aggstate->hashentrysize = hash_agg_entry_size(aggstate->numtrans,
                                                      outerplan->plan_width,
                                                      node->transitionSpace);
 
        /*
         * Consider all of the grouping sets together when setting the limits
         * and estimating the number of partitions. This can be inaccurate
         * when there is more than one grouping set, but should still be
         * reasonable.
         */
        for (int k = 0; k < aggstate->num_hashes; k++)
            totalGroups += aggstate->perhash[k].aggnode->numGroups;
 
        hash_agg_set_limits(aggstate->hashentrysize, totalGroups, 0,
                            &aggstate->hash_mem_limit,
                            &aggstate->hash_ngroups_limit,
                            &aggstate->hash_planned_partitions);
        find_hash_columns(aggstate);
 
        /* Skip massive memory allocation if we are just doing EXPLAIN */
        if (!(eflags & EXEC_FLAG_EXPLAIN_ONLY))
            build_hash_tables(aggstate);
 
        aggstate->table_filled = false;
 
        /* Initialize this to 1, meaning nothing spilled, yet */
        aggstate->hash_batches_used = 1;
    }
 
    /*
     * Initialize current phase-dependent values to initial phase. The initial
     * phase is 1 (first sort pass) for all strategies that use sorting (if
     * hashing is being done too, then phase 0 is processed last); but if only
     * hashing is being done, then phase 0 is all there is.
     */
    if (node->aggstrategy == AGG_HASHED)
    {
        aggstate->current_phase = 0;
        initialize_phase(aggstate, 0);
        select_current_set(aggstate, 0, true);
    }
    else
    {
        aggstate->current_phase = 1;
        initialize_phase(aggstate, 1);
        select_current_set(aggstate, 0, false);
    }
 
    /*
     * Perform lookups of aggregate function info, and initialize the
     * unchanging fields of the per-agg and per-trans data.
     */
    foreach(l, aggstate->aggs)
    {
        Aggref     *aggref = lfirst(l);
        AggStatePerAgg peragg;
        AggStatePerTrans pertrans;
        Oid         aggTransFnInputTypes[FUNC_MAX_ARGS];
        int         numAggTransFnArgs;
        int         numDirectArgs;
        HeapTuple   aggTuple;
        Form_pg_aggregate aggform;
        AclResult   aclresult;
        Oid         finalfn_oid;
        Oid         serialfn_oid,
                    deserialfn_oid;
        Oid         aggOwner;
        Expr       *finalfnexpr;
        Oid         aggtranstype;
 
        /* Planner should have assigned aggregate to correct level */
        Assert(aggref->agglevelsup == 0);
        /* ... and the split mode should match */
        Assert(aggref->aggsplit == aggstate->aggsplit);
 
        peragg = &peraggs[aggref->aggno];
 
        /* Check if we initialized the state for this aggregate already. */
        if (peragg->aggref != NULL)
            continue;
 
        peragg->aggref = aggref;
        peragg->transno = aggref->aggtransno;
 
        /* Fetch the pg_aggregate row */
        aggTuple = SearchSysCache1(AGGFNOID,
                                   ObjectIdGetDatum(aggref->aggfnoid));
        if (!HeapTupleIsValid(aggTuple))
            elog(ERROR, "cache lookup failed for aggregate %u",
                 aggref->aggfnoid);
        aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple);
 
        /* Check permission to call aggregate function */
        aclresult = object_aclcheck(ProcedureRelationId, aggref->aggfnoid, GetUserId(),
                                    ACL_EXECUTE);
        if (aclresult != ACLCHECK_OK)
            aclcheck_error(aclresult, OBJECT_AGGREGATE,
                           get_func_name(aggref->aggfnoid));
        InvokeFunctionExecuteHook(aggref->aggfnoid);
 
        /* planner recorded transition state type in the Aggref itself */
        aggtranstype = aggref->aggtranstype;
        Assert(OidIsValid(aggtranstype));
 
        /* Final function only required if we're finalizing the aggregates */
        if (DO_AGGSPLIT_SKIPFINAL(aggstate->aggsplit))
            peragg->finalfn_oid = finalfn_oid = InvalidOid;
        else
            peragg->finalfn_oid = finalfn_oid = aggform->aggfinalfn;
 
        serialfn_oid = InvalidOid;
        deserialfn_oid = InvalidOid;
 
        /*
         * Check if serialization/deserialization is required.  We only do it
         * for aggregates that have transtype INTERNAL.
         */
        if (aggtranstype == INTERNALOID)
        {
            /*
             * The planner should only have generated a serialize agg node if
             * every aggregate with an INTERNAL state has a serialization
             * function.  Verify that.
             */
            if (DO_AGGSPLIT_SERIALIZE(aggstate->aggsplit))
            {
                /* serialization only valid when not running finalfn */
                Assert(DO_AGGSPLIT_SKIPFINAL(aggstate->aggsplit));
 
                if (!OidIsValid(aggform->aggserialfn))
                    elog(ERROR, "serialfunc not provided for serialization aggregation");
                serialfn_oid = aggform->aggserialfn;
            }
 
            /* Likewise for deserialization functions */
            if (DO_AGGSPLIT_DESERIALIZE(aggstate->aggsplit))
            {
                /* deserialization only valid when combining states */
                Assert(DO_AGGSPLIT_COMBINE(aggstate->aggsplit));
 
                if (!OidIsValid(aggform->aggdeserialfn))
                    elog(ERROR, "deserialfunc not provided for deserialization aggregation");
                deserialfn_oid = aggform->aggdeserialfn;
            }
        }
 
        /* Check that aggregate owner has permission to call component fns */
        {
            HeapTuple   procTuple;
 
            procTuple = SearchSysCache1(PROCOID,
                                        ObjectIdGetDatum(aggref->aggfnoid));
            if (!HeapTupleIsValid(procTuple))
                elog(ERROR, "cache lookup failed for function %u",
                     aggref->aggfnoid);
            aggOwner = ((Form_pg_proc) GETSTRUCT(procTuple))->proowner;
            ReleaseSysCache(procTuple);
 
            if (OidIsValid(finalfn_oid))
            {
                aclresult = object_aclcheck(ProcedureRelationId, finalfn_oid, aggOwner,
                                            ACL_EXECUTE);
                if (aclresult != ACLCHECK_OK)
                    aclcheck_error(aclresult, OBJECT_FUNCTION,
                                   get_func_name(finalfn_oid));
                InvokeFunctionExecuteHook(finalfn_oid);
            }
            if (OidIsValid(serialfn_oid))
            {
                aclresult = object_aclcheck(ProcedureRelationId, serialfn_oid, aggOwner,
                                            ACL_EXECUTE);
                if (aclresult != ACLCHECK_OK)
                    aclcheck_error(aclresult, OBJECT_FUNCTION,
                                   get_func_name(serialfn_oid));
                InvokeFunctionExecuteHook(serialfn_oid);
            }
            if (OidIsValid(deserialfn_oid))
            {
                aclresult = object_aclcheck(ProcedureRelationId, deserialfn_oid, aggOwner,
                                            ACL_EXECUTE);
                if (aclresult != ACLCHECK_OK)
                    aclcheck_error(aclresult, OBJECT_FUNCTION,
                                   get_func_name(deserialfn_oid));
                InvokeFunctionExecuteHook(deserialfn_oid);
            }
        }
 
        /*
         * Get actual datatypes of the (nominal) aggregate inputs.  These
         * could be different from the agg's declared input types, when the
         * agg accepts ANY or a polymorphic type.
         */
        numAggTransFnArgs = get_aggregate_argtypes(aggref,
                                                   aggTransFnInputTypes);
 
        /* Count the "direct" arguments, if any */
        numDirectArgs = list_length(aggref->aggdirectargs);
 
        /* Detect how many arguments to pass to the finalfn */
        if (aggform->aggfinalextra)
            peragg->numFinalArgs = numAggTransFnArgs + 1;
        else
            peragg->numFinalArgs = numDirectArgs + 1;
 
        /* Initialize any direct-argument expressions */
        peragg->aggdirectargs = ExecInitExprList(aggref->aggdirectargs,
                                                 (PlanState *) aggstate);
 
        /*
         * build expression trees using actual argument & result types for the
         * finalfn, if it exists and is required.
         */
        if (OidIsValid(finalfn_oid))
        {
            build_aggregate_finalfn_expr(aggTransFnInputTypes,
                                         peragg->numFinalArgs,
                                         aggtranstype,
                                         aggref->aggtype,
                                         aggref->inputcollid,
                                         finalfn_oid,
                                         &finalfnexpr);
            fmgr_info(finalfn_oid, &peragg->finalfn);
            fmgr_info_set_expr((Node *) finalfnexpr, &peragg->finalfn);
        }
 
        /* get info about the output value's datatype */
        get_typlenbyval(aggref->aggtype,
                        &peragg->resulttypeLen,
                        &peragg->resulttypeByVal);
 
        /*
         * Build working state for invoking the transition function, if we
         * haven't done it already.
         */
        pertrans = &pertransstates[aggref->aggtransno];
        if (pertrans->aggref == NULL)
        {
            Datum       textInitVal;
            Datum       initValue;
            bool        initValueIsNull;
            Oid         transfn_oid;
 
            /*
             * If this aggregation is performing state combines, then instead
             * of using the transition function, we'll use the combine
             * function.
             */
            if (DO_AGGSPLIT_COMBINE(aggstate->aggsplit))
            {
                transfn_oid = aggform->aggcombinefn;
 
                /* If not set then the planner messed up */
                if (!OidIsValid(transfn_oid))
                    elog(ERROR, "combinefn not set for aggregate function");
            }
            else
                transfn_oid = aggform->aggtransfn;
 
            aclresult = object_aclcheck(ProcedureRelationId, transfn_oid, aggOwner, ACL_EXECUTE);
            if (aclresult != ACLCHECK_OK)
                aclcheck_error(aclresult, OBJECT_FUNCTION,
                               get_func_name(transfn_oid));
            InvokeFunctionExecuteHook(transfn_oid);
 
            /*
             * initval is potentially null, so don't try to access it as a
             * struct field. Must do it the hard way with SysCacheGetAttr.
             */
            textInitVal = SysCacheGetAttr(AGGFNOID, aggTuple,
                                          Anum_pg_aggregate_agginitval,
                                          &initValueIsNull);
            if (initValueIsNull)
                initValue = (Datum) 0;
            else
                initValue = GetAggInitVal(textInitVal, aggtranstype);
 
            if (DO_AGGSPLIT_COMBINE(aggstate->aggsplit))
            {
                Oid         combineFnInputTypes[] = {aggtranstype,
                aggtranstype};
 
                /*
                 * When combining there's only one input, the to-be-combined
                 * transition value.  The transition value is not counted
                 * here.
                 */
                pertrans->numTransInputs = 1;
 
                /* aggcombinefn always has two arguments of aggtranstype */
                build_pertrans_for_aggref(pertrans, aggstate, estate,
                                          aggref, transfn_oid, aggtranstype,
                                          serialfn_oid, deserialfn_oid,
                                          initValue, initValueIsNull,
                                          combineFnInputTypes, 2);
 
                /*
                 * Ensure that a combine function to combine INTERNAL states
                 * is not strict. This should have been checked during CREATE
                 * AGGREGATE, but the strict property could have been changed
                 * since then.
                 */
                if (pertrans->transfn.fn_strict && aggtranstype == INTERNALOID)
                    ereport(ERROR,
                            (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION),
                             errmsg("combine function with transition type %s must not be declared STRICT",
                                    format_type_be(aggtranstype))));
            }
            else
            {
                /* Detect how many arguments to pass to the transfn */
                if (AGGKIND_IS_ORDERED_SET(aggref->aggkind))
                    pertrans->numTransInputs = list_length(aggref->args);
                else
                    pertrans->numTransInputs = numAggTransFnArgs;
 
                build_pertrans_for_aggref(pertrans, aggstate, estate,
                                          aggref, transfn_oid, aggtranstype,
                                          serialfn_oid, deserialfn_oid,
                                          initValue, initValueIsNull,
                                          aggTransFnInputTypes,
                                          numAggTransFnArgs);
 
                /*
                 * If the transfn is strict and the initval is NULL, make sure
                 * input type and transtype are the same (or at least
                 * binary-compatible), so that it's OK to use the first
                 * aggregated input value as the initial transValue.  This
                 * should have been checked at agg definition time, but we
                 * must check again in case the transfn's strictness property
                 * has been changed.
                 */
                if (pertrans->transfn.fn_strict && pertrans->initValueIsNull)
                {
                    if (numAggTransFnArgs <= numDirectArgs ||
                        !IsBinaryCoercible(aggTransFnInputTypes[numDirectArgs],
                                           aggtranstype))
                        ereport(ERROR,
                                (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION),
                                 errmsg("aggregate %u needs to have compatible input type and transition type",
                                        aggref->aggfnoid)));
                }
            }
        }
        else
            pertrans->aggshared = true;
        ReleaseSysCache(aggTuple);
    }
 
    /*
     * Last, check whether any more aggregates got added onto the node while
     * we processed the expressions for the aggregate arguments (including not
     * only the regular arguments and FILTER expressions handled immediately
     * above, but any direct arguments we might've handled earlier).  If so,
     * we have nested aggregate functions, which is semantically nonsensical,
     * so complain.  (This should have been caught by the parser, so we don't
     * need to work hard on a helpful error message; but we defend against it
     * here anyway, just to be sure.)
     */
    if (numaggrefs != list_length(aggstate->aggs))
        ereport(ERROR,
                (errcode(ERRCODE_GROUPING_ERROR),
                 errmsg("aggregate function calls cannot be nested")));
 
    /*
     * Build expressions doing all the transition work at once. We build a
     * different one for each phase, as the number of transition function
     * invocation can differ between phases. Note this'll work both for
     * transition and combination functions (although there'll only be one
     * phase in the latter case).
     */
    for (phaseidx = 0; phaseidx < aggstate->numphases; phaseidx++)
    {
        AggStatePerPhase phase = &aggstate->phases[phaseidx];
        bool        dohash = false;
        bool        dosort = false;
 
        /* phase 0 doesn't necessarily exist */
        if (!phase->aggnode)
            continue;
 
        if (aggstate->aggstrategy == AGG_MIXED && phaseidx == 1)
        {
            /*
             * Phase one, and only phase one, in a mixed agg performs both
             * sorting and aggregation.
             */
            dohash = true;
            dosort = true;
        }
        else if (aggstate->aggstrategy == AGG_MIXED && phaseidx == 0)
        {
            /*
             * No need to compute a transition function for an AGG_MIXED phase
             * 0 - the contents of the hashtables will have been computed
             * during phase 1.
             */
            continue;
        }
        else if (phase->aggstrategy == AGG_PLAIN ||
                 phase->aggstrategy == AGG_SORTED)
        {
            dohash = false;
            dosort = true;
        }
        else if (phase->aggstrategy == AGG_HASHED)
        {
            dohash = true;
            dosort = false;
        }
        else
            Assert(false);
 
        phase->evaltrans = ExecBuildAggTrans(aggstate, phase, dosort, dohash,
                                             false);
 
        /* cache compiled expression for outer slot without NULL check */
        phase->evaltrans_cache[0][0] = phase->evaltrans;
    }
 
    return aggstate;
}

Referenced by ExecInitNode().

◆ ExecReScanAgg()

void ExecReScanAgg ( AggState * node )

Definition at line 4466 of file nodeAgg.c.

{
    ExprContext *econtext = node->ss.ps.ps_ExprContext;
    PlanState  *outerPlan = outerPlanState(node);
    Agg        *aggnode = (Agg *) node->ss.ps.plan;
    int         transno;
    int         numGroupingSets = Max(node->maxsets, 1);
    int         setno;
 
    node->agg_done = false;
 
    if (node->aggstrategy == AGG_HASHED)
    {
        /*
         * In the hashed case, if we haven't yet built the hash table then we
         * can just return; nothing done yet, so nothing to undo. If subnode's
         * chgParam is not NULL then it will be re-scanned by ExecProcNode,
         * else no reason to re-scan it at all.
         */
        if (!node->table_filled)
            return;
 
        /*
         * If we do have the hash table, and it never spilled, and the subplan
         * does not have any parameter changes, and none of our own parameter
         * changes affect input expressions of the aggregated functions, then
         * we can just rescan the existing hash table; no need to build it
         * again.
         */
        if (outerPlan->chgParam == NULL && !node->hash_ever_spilled &&
            !bms_overlap(node->ss.ps.chgParam, aggnode->aggParams))
        {
            ResetTupleHashIterator(node->perhash[0].hashtable,
                                   &node->perhash[0].hashiter);
            select_current_set(node, 0, true);
            return;
        }
    }
 
    /* Make sure we have closed any open tuplesorts */
    for (transno = 0; transno < node->numtrans; transno++)
    {
        for (setno = 0; setno < numGroupingSets; setno++)
        {
            AggStatePerTrans pertrans = &node->pertrans[transno];
 
            if (pertrans->sortstates[setno])
            {
                tuplesort_end(pertrans->sortstates[setno]);
                pertrans->sortstates[setno] = NULL;
            }
        }
    }
 
    /*
     * We don't need to ReScanExprContext the output tuple context here;
     * ExecReScan already did it. But we do need to reset our per-grouping-set
     * contexts, which may have transvalues stored in them. (We use rescan
     * rather than just reset because transfns may have registered callbacks
     * that need to be run now.) For the AGG_HASHED case, see below.
     */
 
    for (setno = 0; setno < numGroupingSets; setno++)
    {
        ReScanExprContext(node->aggcontexts[setno]);
    }
 
    /* Release first tuple of group, if we have made a copy */
    if (node->grp_firstTuple != NULL)
    {
        heap_freetuple(node->grp_firstTuple);
        node->grp_firstTuple = NULL;
    }
    ExecClearTuple(node->ss.ss_ScanTupleSlot);
 
    /* Forget current agg values */
    MemSet(econtext->ecxt_aggvalues, 0, sizeof(Datum) * node->numaggs);
    MemSet(econtext->ecxt_aggnulls, 0, sizeof(bool) * node->numaggs);
 
    /*
     * With AGG_HASHED/MIXED, the hash table is allocated in a sub-context of
     * the hashcontext. This used to be an issue, but now, resetting a context
     * automatically deletes sub-contexts too.
     */
    if (node->aggstrategy == AGG_HASHED || node->aggstrategy == AGG_MIXED)
    {
        hashagg_reset_spill_state(node);
 
        node->hash_ever_spilled = false;
        node->hash_spill_mode = false;
        node->hash_ngroups_current = 0;
 
        ReScanExprContext(node->hashcontext);
        MemoryContextReset(node->hash_tablecxt);
        /* Rebuild an empty hash table */
        build_hash_tables(node);
        node->table_filled = false;
        /* iterator will be reset when the table is filled */
 
        hashagg_recompile_expressions(node, false, false);
    }
 
    if (node->aggstrategy != AGG_HASHED)
    {
        /*
         * Reset the per-group state (in particular, mark transvalues null)
         */
        for (setno = 0; setno < numGroupingSets; setno++)
        {
            MemSet(node->pergroups[setno], 0,
                   sizeof(AggStatePerGroupData) * node->numaggs);
        }
 
        /* reset to phase 1 */
        initialize_phase(node, 1);
 
        node->input_done = false;
        node->projected_set = -1;
    }
 
    if (outerPlan->chgParam == NULL)
        ExecReScan(outerPlan);
}

References AggState::agg_done, AGG_HASHED, AGG_MIXED, AggState::aggcontexts, Agg::aggParams, AggState::aggstrategy, bms_overlap(), build_hash_tables(), PlanState::chgParam, ExprContext::ecxt_aggnulls, ExprContext::ecxt_aggvalues, ExecClearTuple(), ExecReScan(), AggState::grp_firstTuple, AggState::hash_ever_spilled, AggState::hash_ngroups_current, AggState::hash_spill_mode, AggState::hash_tablecxt, hashagg_recompile_expressions(), hashagg_reset_spill_state(), AggState::hashcontext, AggStatePerHashData::hashiter, AggStatePerHashData::hashtable, heap_freetuple(), initialize_phase(), AggState::input_done, Max, AggState::maxsets, MemoryContextReset(), MemSet, AggState::numaggs, AggState::numtrans, outerPlan, outerPlanState, AggState::pergroups, AggState::perhash, AggState::pertrans, PlanState::plan, AggState::projected_set, ScanState::ps, PlanState::ps_ExprContext, ReScanExprContext(), ResetTupleHashIterator, select_current_set(), AggStatePerTransData::sortstates, AggState::ss, ScanState::ss_ScanTupleSlot, AggState::table_filled, and tuplesort_end().

Referenced by ExecReScan().

◆ fetch_input_tuple()

static TupleTableSlot * fetch_input_tuple ( AggState * aggstate )

static

Definition at line 549 of file nodeAgg.c.

{
    TupleTableSlot *slot;
 
    if (aggstate->sort_in)
    {
        /* make sure we check for interrupts in either path through here */
        CHECK_FOR_INTERRUPTS();
        if (!tuplesort_gettupleslot(aggstate->sort_in, true, false,
                                    aggstate->sort_slot, NULL))
            return NULL;
        slot = aggstate->sort_slot;
    }
    else
        slot = ExecProcNode(outerPlanState(aggstate));
 
    if (!TupIsNull(slot) && aggstate->sort_out)
        tuplesort_puttupleslot(aggstate->sort_out, slot);
 
    return slot;
}

References CHECK_FOR_INTERRUPTS, ExecProcNode(), outerPlanState, AggState::sort_in, AggState::sort_out, AggState::sort_slot, TupIsNull, tuplesort_gettupleslot(), and tuplesort_puttupleslot().

Referenced by agg_fill_hash_table(), and agg_retrieve_direct().

◆ finalize_aggregate()

static void finalize_aggregate	(	AggState *	aggstate,
		AggStatePerAgg	peragg,
		AggStatePerGroup	pergroupstate,
		Datum *	resultVal,
		bool *	resultIsNull
	)

static

Definition at line 1045 of file nodeAgg.c.

{
    LOCAL_FCINFO(fcinfo, FUNC_MAX_ARGS);
    bool        anynull = false;
    MemoryContext oldContext;
    int         i;
    ListCell   *lc;
    AggStatePerTrans pertrans = &aggstate->pertrans[peragg->transno];
 
    oldContext = MemoryContextSwitchTo(aggstate->ss.ps.ps_ExprContext->ecxt_per_tuple_memory);
 
    /*
     * Evaluate any direct arguments.  We do this even if there's no finalfn
     * (which is unlikely anyway), so that side-effects happen as expected.
     * The direct arguments go into arg positions 1 and up, leaving position 0
     * for the transition state value.
     */
    i = 1;
    foreach(lc, peragg->aggdirectargs)
    {
        ExprState  *expr = (ExprState *) lfirst(lc);
 
        fcinfo->args[i].value = ExecEvalExpr(expr,
                                             aggstate->ss.ps.ps_ExprContext,
                                             &fcinfo->args[i].isnull);
        anynull |= fcinfo->args[i].isnull;
        i++;
    }
 
    /*
     * Apply the agg's finalfn if one is provided, else return transValue.
     */
    if (OidIsValid(peragg->finalfn_oid))
    {
        int         numFinalArgs = peragg->numFinalArgs;
 
        /* set up aggstate->curperagg for AggGetAggref() */
        aggstate->curperagg = peragg;
 
        InitFunctionCallInfoData(*fcinfo, &peragg->finalfn,
                                 numFinalArgs,
                                 pertrans->aggCollation,
                                 (Node *) aggstate, NULL);
 
        /* Fill in the transition state value */
        fcinfo->args[0].value =
            MakeExpandedObjectReadOnly(pergroupstate->transValue,
                                       pergroupstate->transValueIsNull,
                                       pertrans->transtypeLen);
        fcinfo->args[0].isnull = pergroupstate->transValueIsNull;
        anynull |= pergroupstate->transValueIsNull;
 
        /* Fill any remaining argument positions with nulls */
        for (; i < numFinalArgs; i++)
        {
            fcinfo->args[i].value = (Datum) 0;
            fcinfo->args[i].isnull = true;
            anynull = true;
        }
 
        if (fcinfo->flinfo->fn_strict && anynull)
        {
            /* don't call a strict function with NULL inputs */
            *resultVal = (Datum) 0;
            *resultIsNull = true;
        }
        else
        {
            Datum       result;
 
            result = FunctionCallInvoke(fcinfo);
            *resultIsNull = fcinfo->isnull;
            *resultVal = MakeExpandedObjectReadOnly(result,
                                                    fcinfo->isnull,
                                                    peragg->resulttypeLen);
        }
        aggstate->curperagg = NULL;
    }
    else
    {
        *resultVal =
            MakeExpandedObjectReadOnly(pergroupstate->transValue,
                                       pergroupstate->transValueIsNull,
                                       pertrans->transtypeLen);
        *resultIsNull = pergroupstate->transValueIsNull;
    }
 
    MemoryContextSwitchTo(oldContext);
}

References AggStatePerTransData::aggCollation, AggStatePerAggData::aggdirectargs, AggState::curperagg, ExprContext::ecxt_per_tuple_memory, ExecEvalExpr(), AggStatePerAggData::finalfn, AggStatePerAggData::finalfn_oid, FUNC_MAX_ARGS, FunctionCallInvoke, i, InitFunctionCallInfoData, lfirst, LOCAL_FCINFO, MakeExpandedObjectReadOnly, MemoryContextSwitchTo(), AggStatePerAggData::numFinalArgs, OidIsValid, AggState::pertrans, ScanState::ps, PlanState::ps_ExprContext, AggStatePerAggData::resulttypeLen, AggState::ss, AggStatePerAggData::transno, AggStatePerTransData::transtypeLen, AggStatePerGroupData::transValue, and AggStatePerGroupData::transValueIsNull.

Referenced by finalize_aggregates().

◆ finalize_aggregates()

static void finalize_aggregates	(	AggState *	aggstate,
		AggStatePerAgg	peraggs,
		AggStatePerGroup	pergroup
	)

static

Definition at line 1293 of file nodeAgg.c.

{
    ExprContext *econtext = aggstate->ss.ps.ps_ExprContext;
    Datum      *aggvalues = econtext->ecxt_aggvalues;
    bool       *aggnulls = econtext->ecxt_aggnulls;
    int         aggno;
 
    /*
     * If there were any DISTINCT and/or ORDER BY aggregates, sort their
     * inputs and run the transition functions.
     */
    for (int transno = 0; transno < aggstate->numtrans; transno++)
    {
        AggStatePerTrans pertrans = &aggstate->pertrans[transno];
        AggStatePerGroup pergroupstate;
 
        pergroupstate = &pergroup[transno];
 
        if (pertrans->aggsortrequired)
        {
            Assert(aggstate->aggstrategy != AGG_HASHED &&
                   aggstate->aggstrategy != AGG_MIXED);
 
            if (pertrans->numInputs == 1)
                process_ordered_aggregate_single(aggstate,
                                                 pertrans,
                                                 pergroupstate);
            else
                process_ordered_aggregate_multi(aggstate,
                                                pertrans,
                                                pergroupstate);
        }
        else if (pertrans->numDistinctCols > 0 && pertrans->haslast)
        {
            pertrans->haslast = false;
 
            if (pertrans->numDistinctCols == 1)
            {
                if (!pertrans->inputtypeByVal && !pertrans->lastisnull)
                    pfree(DatumGetPointer(pertrans->lastdatum));
 
                pertrans->lastisnull = false;
                pertrans->lastdatum = (Datum) 0;
            }
            else
                ExecClearTuple(pertrans->uniqslot);
        }
    }
 
    /*
     * Run the final functions.
     */
    for (aggno = 0; aggno < aggstate->numaggs; aggno++)
    {
        AggStatePerAgg peragg = &peraggs[aggno];
        int         transno = peragg->transno;
        AggStatePerGroup pergroupstate;
 
        pergroupstate = &pergroup[transno];
 
        if (DO_AGGSPLIT_SKIPFINAL(aggstate->aggsplit))
            finalize_partialaggregate(aggstate, peragg, pergroupstate,
                                      &aggvalues[aggno], &aggnulls[aggno]);
        else
            finalize_aggregate(aggstate, peragg, pergroupstate,
                               &aggvalues[aggno], &aggnulls[aggno]);
    }
}

Referenced by agg_retrieve_direct(), and agg_retrieve_hash_table_in_memory().

◆ finalize_partialaggregate()

static void finalize_partialaggregate	(	AggState *	aggstate,
		AggStatePerAgg	peragg,
		AggStatePerGroup	pergroupstate,
		Datum *	resultVal,
		bool *	resultIsNull
	)

static

Definition at line 1145 of file nodeAgg.c.

{
    AggStatePerTrans pertrans = &aggstate->pertrans[peragg->transno];
    MemoryContext oldContext;
 
    oldContext = MemoryContextSwitchTo(aggstate->ss.ps.ps_ExprContext->ecxt_per_tuple_memory);
 
    /*
     * serialfn_oid will be set if we must serialize the transvalue before
     * returning it
     */
    if (OidIsValid(pertrans->serialfn_oid))
    {
        /* Don't call a strict serialization function with NULL input. */
        if (pertrans->serialfn.fn_strict && pergroupstate->transValueIsNull)
        {
            *resultVal = (Datum) 0;
            *resultIsNull = true;
        }
        else
        {
            FunctionCallInfo fcinfo = pertrans->serialfn_fcinfo;
            Datum       result;
 
            fcinfo->args[0].value =
                MakeExpandedObjectReadOnly(pergroupstate->transValue,
                                           pergroupstate->transValueIsNull,
                                           pertrans->transtypeLen);
            fcinfo->args[0].isnull = pergroupstate->transValueIsNull;
            fcinfo->isnull = false;
 
            result = FunctionCallInvoke(fcinfo);
            *resultIsNull = fcinfo->isnull;
            *resultVal = MakeExpandedObjectReadOnly(result,
                                                    fcinfo->isnull,
                                                    peragg->resulttypeLen);
        }
    }
    else
    {
        *resultVal =
            MakeExpandedObjectReadOnly(pergroupstate->transValue,
                                       pergroupstate->transValueIsNull,
                                       pertrans->transtypeLen);
        *resultIsNull = pergroupstate->transValueIsNull;
    }
 
    MemoryContextSwitchTo(oldContext);
}

References FunctionCallInfoBaseData::args, ExprContext::ecxt_per_tuple_memory, FmgrInfo::fn_strict, FunctionCallInvoke, FunctionCallInfoBaseData::isnull, NullableDatum::isnull, MakeExpandedObjectReadOnly, MemoryContextSwitchTo(), OidIsValid, AggState::pertrans, ScanState::ps, PlanState::ps_ExprContext, AggStatePerAggData::resulttypeLen, AggStatePerTransData::serialfn, AggStatePerTransData::serialfn_fcinfo, AggStatePerTransData::serialfn_oid, AggState::ss, AggStatePerAggData::transno, AggStatePerTransData::transtypeLen, AggStatePerGroupData::transValue, AggStatePerGroupData::transValueIsNull, and NullableDatum::value.

Referenced by finalize_aggregates().

◆ find_cols()

static void find_cols	(	AggState *	aggstate,
		Bitmapset **	aggregated,
		Bitmapset **	unaggregated
	)

static

Definition at line 1396 of file nodeAgg.c.

{
    Agg        *agg = (Agg *) aggstate->ss.ps.plan;
    FindColsContext context;
 
    context.is_aggref = false;
    context.aggregated = NULL;
    context.unaggregated = NULL;
 
    /* Examine tlist and quals */
    (void) find_cols_walker((Node *) agg->plan.targetlist, &context);
    (void) find_cols_walker((Node *) agg->plan.qual, &context);
 
    /* In some cases, grouping columns will not appear in the tlist */
    for (int i = 0; i < agg->numCols; i++)
        context.unaggregated = bms_add_member(context.unaggregated,
                                              agg->grpColIdx[i]);
 
    *aggregated = context.aggregated;
    *unaggregated = context.unaggregated;
}

References bms_add_member(), find_cols_walker(), i, FindColsContext::is_aggref, Agg::numCols, PlanState::plan, Agg::plan, ScanState::ps, Plan::qual, AggState::ss, and Plan::targetlist.

Referenced by find_hash_columns().

◆ find_cols_walker()

static bool find_cols_walker	(	Node *	node,
		FindColsContext *	context
	)

static

Definition at line 1419 of file nodeAgg.c.

{
    if (node == NULL)
        return false;
    if (IsA(node, Var))
    {
        Var        *var = (Var *) node;
 
        /* setrefs.c should have set the varno to OUTER_VAR */
        Assert(var->varno == OUTER_VAR);
        Assert(var->varlevelsup == 0);
        if (context->is_aggref)
            context->aggregated = bms_add_member(context->aggregated,
                                                 var->varattno);
        else
            context->unaggregated = bms_add_member(context->unaggregated,
                                                   var->varattno);
        return false;
    }
    if (IsA(node, Aggref))
    {
        Assert(!context->is_aggref);
        context->is_aggref = true;
        expression_tree_walker(node, find_cols_walker, context);
        context->is_aggref = false;
        return false;
    }
    return expression_tree_walker(node, find_cols_walker, context);
}

References FindColsContext::aggregated, Assert(), bms_add_member(), expression_tree_walker, find_cols_walker(), FindColsContext::is_aggref, IsA, OUTER_VAR, FindColsContext::unaggregated, Var::varattno, Var::varlevelsup, and Var::varno.

Referenced by find_cols(), and find_cols_walker().

◆ find_hash_columns()

static void find_hash_columns ( AggState * aggstate )

static

Definition at line 1570 of file nodeAgg.c.

{
    Bitmapset  *base_colnos;
    Bitmapset  *aggregated_colnos;
    TupleDesc   scanDesc = aggstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor;
    List       *outerTlist = outerPlanState(aggstate)->plan->targetlist;
    int         numHashes = aggstate->num_hashes;
    EState     *estate = aggstate->ss.ps.state;
    int         j;
 
    /* Find Vars that will be needed in tlist and qual */
    find_cols(aggstate, &aggregated_colnos, &base_colnos);
    aggstate->colnos_needed = bms_union(base_colnos, aggregated_colnos);
    aggstate->max_colno_needed = 0;
    aggstate->all_cols_needed = true;
 
    for (int i = 0; i < scanDesc->natts; i++)
    {
        int         colno = i + 1;
 
        if (bms_is_member(colno, aggstate->colnos_needed))
            aggstate->max_colno_needed = colno;
        else
            aggstate->all_cols_needed = false;
    }
 
    for (j = 0; j < numHashes; ++j)
    {
        AggStatePerHash perhash = &aggstate->perhash[j];
        Bitmapset  *colnos = bms_copy(base_colnos);
        AttrNumber *grpColIdx = perhash->aggnode->grpColIdx;
        List       *hashTlist = NIL;
        TupleDesc   hashDesc;
        int         maxCols;
        int         i;
 
        perhash->largestGrpColIdx = 0;
 
        /*
         * If we're doing grouping sets, then some Vars might be referenced in
         * tlist/qual for the benefit of other grouping sets, but not needed
         * when hashing; i.e. prepare_projection_slot will null them out, so
         * there'd be no point storing them.  Use prepare_projection_slot's
         * logic to determine which.
         */
        if (aggstate->phases[0].grouped_cols)
        {
            Bitmapset  *grouped_cols = aggstate->phases[0].grouped_cols[j];
            ListCell   *lc;
 
            foreach(lc, aggstate->all_grouped_cols)
            {
                int         attnum = lfirst_int(lc);
 
                if (!bms_is_member(attnum, grouped_cols))
                    colnos = bms_del_member(colnos, attnum);
            }
        }
 
        /*
         * Compute maximum number of input columns accounting for possible
         * duplications in the grpColIdx array, which can happen in some edge
         * cases where HashAggregate was generated as part of a semijoin or a
         * DISTINCT.
         */
        maxCols = bms_num_members(colnos) + perhash->numCols;
 
        perhash->hashGrpColIdxInput =
            palloc(maxCols * sizeof(AttrNumber));
        perhash->hashGrpColIdxHash =
            palloc(perhash->numCols * sizeof(AttrNumber));
 
        /* Add all the grouping columns to colnos */
        for (i = 0; i < perhash->numCols; i++)
            colnos = bms_add_member(colnos, grpColIdx[i]);
 
        /*
         * First build mapping for columns directly hashed. These are the
         * first, because they'll be accessed when computing hash values and
         * comparing tuples for exact matches. We also build simple mapping
         * for execGrouping, so it knows where to find the to-be-hashed /
         * compared columns in the input.
         */
        for (i = 0; i < perhash->numCols; i++)
        {
            perhash->hashGrpColIdxInput[i] = grpColIdx[i];
            perhash->hashGrpColIdxHash[i] = i + 1;
            perhash->numhashGrpCols++;
            /* delete already mapped columns */
            colnos = bms_del_member(colnos, grpColIdx[i]);
        }
 
        /* and add the remaining columns */
        i = -1;
        while ((i = bms_next_member(colnos, i)) >= 0)
        {
            perhash->hashGrpColIdxInput[perhash->numhashGrpCols] = i;
            perhash->numhashGrpCols++;
        }
 
        /* and build a tuple descriptor for the hashtable */
        for (i = 0; i < perhash->numhashGrpCols; i++)
        {
            int         varNumber = perhash->hashGrpColIdxInput[i] - 1;
 
            hashTlist = lappend(hashTlist, list_nth(outerTlist, varNumber));
            perhash->largestGrpColIdx =
                Max(varNumber + 1, perhash->largestGrpColIdx);
        }
 
        hashDesc = ExecTypeFromTL(hashTlist);
 
        execTuplesHashPrepare(perhash->numCols,
                              perhash->aggnode->grpOperators,
                              &perhash->eqfuncoids,
                              &perhash->hashfunctions);
        perhash->hashslot =
            ExecAllocTableSlot(&estate->es_tupleTable, hashDesc,
                               &TTSOpsMinimalTuple);
 
        list_free(hashTlist);
        bms_free(colnos);
    }
 
    bms_free(base_colnos);
}

Referenced by ExecInitAgg().

◆ GetAggInitVal()

static Datum GetAggInitVal	(	Datum	textInitVal,
		Oid	transtype
	)

static

Definition at line 4384 of file nodeAgg.c.

{
    Oid         typinput,
                typioparam;
    char       *strInitVal;
    Datum       initVal;
 
    getTypeInputInfo(transtype, &typinput, &typioparam);
    strInitVal = TextDatumGetCString(textInitVal);
    initVal = OidInputFunctionCall(typinput, strInitVal,
                                   typioparam, -1);
    pfree(strInitVal);
    return initVal;
}

References getTypeInputInfo(), OidInputFunctionCall(), pfree(), and TextDatumGetCString.

Referenced by ExecInitAgg().

◆ hash_agg_check_limits()

static void hash_agg_check_limits ( AggState * aggstate )

static

Definition at line 1867 of file nodeAgg.c.

{
    uint64      ngroups = aggstate->hash_ngroups_current;
    Size        meta_mem = MemoryContextMemAllocated(aggstate->hash_metacxt,
                                                     true);
    Size        entry_mem = MemoryContextMemAllocated(aggstate->hash_tablecxt,
                                                      true);
    Size        tval_mem = MemoryContextMemAllocated(aggstate->hashcontext->ecxt_per_tuple_memory,
                                                     true);
    Size        total_mem = meta_mem + entry_mem + tval_mem;
    bool        do_spill = false;
 
#ifdef USE_INJECTION_POINTS
    if (ngroups >= 1000)
    {
        if (IS_INJECTION_POINT_ATTACHED("hash-aggregate-spill-1000"))
        {
            do_spill = true;
            INJECTION_POINT_CACHED("hash-aggregate-spill-1000");
        }
    }
#endif
 
    /*
     * Don't spill unless there's at least one group in the hash table so we
     * can be sure to make progress even in edge cases.
     */
    if (aggstate->hash_ngroups_current > 0 &&
        (total_mem > aggstate->hash_mem_limit ||
         ngroups > aggstate->hash_ngroups_limit))
    {
        do_spill = true;
    }
 
    if (do_spill)
        hash_agg_enter_spill_mode(aggstate);
}

References ExprContext::ecxt_per_tuple_memory, hash_agg_enter_spill_mode(), AggState::hash_mem_limit, AggState::hash_metacxt, AggState::hash_ngroups_current, AggState::hash_ngroups_limit, AggState::hash_tablecxt, AggState::hashcontext, INJECTION_POINT_CACHED, IS_INJECTION_POINT_ATTACHED, and MemoryContextMemAllocated().

Referenced by initialize_hash_entry().

◆ hash_agg_enter_spill_mode()

static void hash_agg_enter_spill_mode ( AggState * aggstate )

static

Definition at line 1911 of file nodeAgg.c.

{
    INJECTION_POINT("hash-aggregate-enter-spill-mode");
    aggstate->hash_spill_mode = true;
    hashagg_recompile_expressions(aggstate, aggstate->table_filled, true);
 
    if (!aggstate->hash_ever_spilled)
    {
        Assert(aggstate->hash_tapeset == NULL);
        Assert(aggstate->hash_spills == NULL);
 
        aggstate->hash_ever_spilled = true;
 
        aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
 
        aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
 
        for (int setno = 0; setno < aggstate->num_hashes; setno++)
        {
            AggStatePerHash perhash = &aggstate->perhash[setno];
            HashAggSpill *spill = &aggstate->hash_spills[setno];
 
            hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
                               perhash->aggnode->numGroups,
                               aggstate->hashentrysize);
        }
    }
}

References AggStatePerHashData::aggnode, Assert(), AggState::hash_ever_spilled, AggState::hash_spill_mode, AggState::hash_spills, AggState::hash_tapeset, hashagg_recompile_expressions(), hashagg_spill_init(), AggState::hashentrysize, INJECTION_POINT, LogicalTapeSetCreate(), AggState::num_hashes, Agg::numGroups, palloc(), AggState::perhash, and AggState::table_filled.

Referenced by hash_agg_check_limits().

◆ hash_agg_entry_size()

Size hash_agg_entry_size	(	int	numTrans,
		Size	tupleWidth,
		Size	transitionSpace
	)

Definition at line 1701 of file nodeAgg.c.

{
    Size        tupleChunkSize;
    Size        pergroupChunkSize;
    Size        transitionChunkSize;
    Size        tupleSize = (MAXALIGN(SizeofMinimalTupleHeader) +
                             tupleWidth);
    Size        pergroupSize = numTrans * sizeof(AggStatePerGroupData);
 
    /*
     * Entries use the Bump allocator, so the chunk sizes are the same as the
     * requested sizes.
     */
    tupleChunkSize = MAXALIGN(tupleSize);
    pergroupChunkSize = pergroupSize;
 
    /*
     * Transition values use AllocSet, which has a chunk header and also uses
     * power-of-two allocations.
     */
    if (transitionSpace > 0)
        transitionChunkSize = CHUNKHDRSZ + pg_nextpower2_size_t(transitionSpace);
    else
        transitionChunkSize = 0;
 
    return
        TupleHashEntrySize() +
        tupleChunkSize +
        pergroupChunkSize +
        transitionChunkSize;
}

References CHUNKHDRSZ, MAXALIGN, pg_nextpower2_size_t, SizeofMinimalTupleHeader, and TupleHashEntrySize().

Referenced by cost_agg(), estimate_hashagg_tablesize(), and ExecInitAgg().

◆ hash_agg_set_limits()

void hash_agg_set_limits	(	double	hashentrysize,
		double	input_groups,
		int	used_bits,
		Size *	mem_limit,
		uint64 *	ngroups_limit,
		int *	num_partitions
	)

Definition at line 1809 of file nodeAgg.c.

{
    int         npartitions;
    Size        partition_mem;
    Size        hash_mem_limit = get_hash_memory_limit();
 
    /* if not expected to spill, use all of hash_mem */
    if (input_groups * hashentrysize <= hash_mem_limit)
    {
        if (num_partitions != NULL)
            *num_partitions = 0;
        *mem_limit = hash_mem_limit;
        *ngroups_limit = hash_mem_limit / hashentrysize;
        return;
    }
 
    /*
     * Calculate expected memory requirements for spilling, which is the size
     * of the buffers needed for all the tapes that need to be open at once.
     * Then, subtract that from the memory available for holding hash tables.
     */
    npartitions = hash_choose_num_partitions(input_groups,
                                             hashentrysize,
                                             used_bits,
                                             NULL);
    if (num_partitions != NULL)
        *num_partitions = npartitions;
 
    partition_mem =
        HASHAGG_READ_BUFFER_SIZE +
        HASHAGG_WRITE_BUFFER_SIZE * npartitions;
 
    /*
     * Don't set the limit below 3/4 of hash_mem. In that case, we are at the
     * minimum number of partitions, so we aren't going to dramatically exceed
     * work mem anyway.
     */
    if (hash_mem_limit > 4 * partition_mem)
        *mem_limit = hash_mem_limit - partition_mem;
    else
        *mem_limit = hash_mem_limit * 0.75;
 
    if (*mem_limit > hashentrysize)
        *ngroups_limit = *mem_limit / hashentrysize;
    else
        *ngroups_limit = 1;
}

References get_hash_memory_limit(), hash_choose_num_partitions(), HASHAGG_READ_BUFFER_SIZE, and HASHAGG_WRITE_BUFFER_SIZE.

Referenced by agg_refill_hash_table(), cost_agg(), and ExecInitAgg().

◆ hash_agg_update_metrics()

static void hash_agg_update_metrics	(	AggState *	aggstate,
		bool	from_tape,
		int	npartitions
	)

static

Definition at line 1947 of file nodeAgg.c.

{
    Size        meta_mem;
    Size        entry_mem;
    Size        hashkey_mem;
    Size        buffer_mem;
    Size        total_mem;
 
    if (aggstate->aggstrategy != AGG_MIXED &&
        aggstate->aggstrategy != AGG_HASHED)
        return;
 
    /* memory for the hash table itself */
    meta_mem = MemoryContextMemAllocated(aggstate->hash_metacxt, true);
 
    /* memory for hash entries */
    entry_mem = MemoryContextMemAllocated(aggstate->hash_tablecxt, true);
 
    /* memory for byref transition states */
    hashkey_mem = MemoryContextMemAllocated(aggstate->hashcontext->ecxt_per_tuple_memory, true);
 
    /* memory for read/write tape buffers, if spilled */
    buffer_mem = npartitions * HASHAGG_WRITE_BUFFER_SIZE;
    if (from_tape)
        buffer_mem += HASHAGG_READ_BUFFER_SIZE;
 
    /* update peak mem */
    total_mem = meta_mem + entry_mem + hashkey_mem + buffer_mem;
    if (total_mem > aggstate->hash_mem_peak)
        aggstate->hash_mem_peak = total_mem;
 
    /* update disk usage */
    if (aggstate->hash_tapeset != NULL)
    {
        uint64      disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
 
        if (aggstate->hash_disk_used < disk_used)
            aggstate->hash_disk_used = disk_used;
    }
 
    /* update hashentrysize estimate based on contents */
    if (aggstate->hash_ngroups_current > 0)
    {
        aggstate->hashentrysize =
            TupleHashEntrySize() +
            (hashkey_mem / (double) aggstate->hash_ngroups_current);
    }
}

References AGG_HASHED, AGG_MIXED, AggState::aggstrategy, ExprContext::ecxt_per_tuple_memory, AggState::hash_disk_used, AggState::hash_mem_peak, AggState::hash_metacxt, AggState::hash_ngroups_current, AggState::hash_tablecxt, AggState::hash_tapeset, HASHAGG_READ_BUFFER_SIZE, HASHAGG_WRITE_BUFFER_SIZE, AggState::hashcontext, AggState::hashentrysize, LogicalTapeSetBlocks(), MemoryContextMemAllocated(), and TupleHashEntrySize().

Referenced by agg_refill_hash_table(), and hashagg_finish_initial_spills().

◆ hash_choose_num_buckets()

static long hash_choose_num_buckets	(	double	hashentrysize,
		long	ngroups,
		Size	memory
	)

static

Definition at line 2058 of file nodeAgg.c.

{
    long        max_nbuckets;
    long        nbuckets = ngroups;
 
    max_nbuckets = memory / hashentrysize;
 
    /*
     * Underestimating is better than overestimating. Too many buckets crowd
     * out space for group keys and transition state values.
     */
    max_nbuckets >>= 1;
 
    if (nbuckets > max_nbuckets)
        nbuckets = max_nbuckets;
 
    return Max(nbuckets, 1);
}

References Max.

Referenced by build_hash_tables().

◆ hash_choose_num_partitions()

static int hash_choose_num_partitions	(	double	input_groups,
		double	hashentrysize,
		int	used_bits,
		int *	log2_npartitions
	)

static

Definition at line 2083 of file nodeAgg.c.

{
    Size        hash_mem_limit = get_hash_memory_limit();
    double      partition_limit;
    double      mem_wanted;
    double      dpartitions;
    int         npartitions;
    int         partition_bits;
 
    /*
     * Avoid creating so many partitions that the memory requirements of the
     * open partition files are greater than 1/4 of hash_mem.
     */
    partition_limit =
        (hash_mem_limit * 0.25 - HASHAGG_READ_BUFFER_SIZE) /
        HASHAGG_WRITE_BUFFER_SIZE;
 
    mem_wanted = HASHAGG_PARTITION_FACTOR * input_groups * hashentrysize;
 
    /* make enough partitions so that each one is likely to fit in memory */
    dpartitions = 1 + (mem_wanted / hash_mem_limit);
 
    if (dpartitions > partition_limit)
        dpartitions = partition_limit;
 
    if (dpartitions < HASHAGG_MIN_PARTITIONS)
        dpartitions = HASHAGG_MIN_PARTITIONS;
    if (dpartitions > HASHAGG_MAX_PARTITIONS)
        dpartitions = HASHAGG_MAX_PARTITIONS;
 
    /* HASHAGG_MAX_PARTITIONS limit makes this safe */
    npartitions = (int) dpartitions;
 
    /* ceil(log2(npartitions)) */
    partition_bits = my_log2(npartitions);
 
    /* make sure that we don't exhaust the hash bits */
    if (partition_bits + used_bits >= 32)
        partition_bits = 32 - used_bits;
 
    if (log2_npartitions != NULL)
        *log2_npartitions = partition_bits;
 
    /* number of partitions will be a power of two */
    npartitions = 1 << partition_bits;
 
    return npartitions;
}

References get_hash_memory_limit(), HASHAGG_MAX_PARTITIONS, HASHAGG_MIN_PARTITIONS, HASHAGG_PARTITION_FACTOR, HASHAGG_READ_BUFFER_SIZE, HASHAGG_WRITE_BUFFER_SIZE, and my_log2().

Referenced by hash_agg_set_limits(), and hashagg_spill_init().

◆ hash_create_memory()

static void hash_create_memory ( AggState * aggstate )

static

Definition at line 2000 of file nodeAgg.c.

{
    Size        maxBlockSize = ALLOCSET_DEFAULT_MAXSIZE;
 
    /*
     * The hashcontext's per-tuple memory will be used for byref transition
     * values and returned by AggCheckCallContext().
     */
    aggstate->hashcontext = CreateWorkExprContext(aggstate->ss.ps.state);
 
    /*
     * The meta context will be used for the bucket array of
     * TupleHashEntryData (or arrays, in the case of grouping sets). As the
     * hash table grows, the bucket array will double in size and the old one
     * will be freed, so an AllocSet is appropriate. For large bucket arrays,
     * the large allocation path will be used, so it's not worth worrying
     * about wasting space due to power-of-two allocations.
     */
    aggstate->hash_metacxt = AllocSetContextCreate(aggstate->ss.ps.state->es_query_cxt,
                                                   "HashAgg meta context",
                                                   ALLOCSET_DEFAULT_SIZES);
 
    /*
     * The hash entries themselves, which include the grouping key
     * (firstTuple) and pergroup data, are stored in the table context. The
     * bump allocator can be used because the entries are not freed until the
     * entire hash table is reset. The bump allocator is faster for
     * allocations and avoids wasting space on the chunk header or
     * power-of-two allocations.
     *
     * Like CreateWorkExprContext(), use smaller sizings for smaller work_mem,
     * to avoid large jumps in memory usage.
     */
 
    /*
     * Like CreateWorkExprContext(), use smaller sizings for smaller work_mem,
     * to avoid large jumps in memory usage.
     */
    maxBlockSize = pg_prevpower2_size_t(work_mem * (Size) 1024 / 16);
 
    /* But no bigger than ALLOCSET_DEFAULT_MAXSIZE */
    maxBlockSize = Min(maxBlockSize, ALLOCSET_DEFAULT_MAXSIZE);
 
    /* and no smaller than ALLOCSET_DEFAULT_INITSIZE */
    maxBlockSize = Max(maxBlockSize, ALLOCSET_DEFAULT_INITSIZE);
 
    aggstate->hash_tablecxt = BumpContextCreate(aggstate->ss.ps.state->es_query_cxt,
                                                "HashAgg table context",
                                                ALLOCSET_DEFAULT_MINSIZE,
                                                ALLOCSET_DEFAULT_INITSIZE,
                                                maxBlockSize);
 
}

References ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate, BumpContextCreate(), CreateWorkExprContext(), EState::es_query_cxt, AggState::hash_metacxt, AggState::hash_tablecxt, AggState::hashcontext, Max, Min, pg_prevpower2_size_t, ScanState::ps, AggState::ss, PlanState::state, and work_mem.

Referenced by ExecInitAgg().

◆ hashagg_batch_new()

static HashAggBatch * hashagg_batch_new	(	LogicalTape *	input_tape,
		int	setno,
		int64	input_tuples,
		double	input_card,
		int	used_bits
	)

static

Definition at line 3097 of file nodeAgg.c.

{
    HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
 
    batch->setno = setno;
    batch->used_bits = used_bits;
    batch->input_tape = input_tape;
    batch->input_tuples = input_tuples;
    batch->input_card = input_card;
 
    return batch;
}

References HashAggBatch::input_card, HashAggBatch::input_tape, HashAggBatch::input_tuples, palloc0(), HashAggBatch::setno, and HashAggBatch::used_bits.

Referenced by hashagg_spill_finish().

◆ hashagg_batch_read()

static MinimalTuple hashagg_batch_read	(	HashAggBatch *	batch,
		uint32 *	hashp
	)

static

Definition at line 3116 of file nodeAgg.c.

{
    LogicalTape *tape = batch->input_tape;
    MinimalTuple tuple;
    uint32      t_len;
    size_t      nread;
    uint32      hash;
 
    nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
    if (nread == 0)
        return NULL;
    if (nread != sizeof(uint32))
        ereport(ERROR,
                (errcode_for_file_access(),
                 errmsg_internal("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
                                 tape, sizeof(uint32), nread)));
    if (hashp != NULL)
        *hashp = hash;
 
    nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
    if (nread != sizeof(uint32))
        ereport(ERROR,
                (errcode_for_file_access(),
                 errmsg_internal("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
                                 tape, sizeof(uint32), nread)));
 
    tuple = (MinimalTuple) palloc(t_len);
    tuple->t_len = t_len;
 
    nread = LogicalTapeRead(tape,
                            (char *) tuple + sizeof(uint32),
                            t_len - sizeof(uint32));
    if (nread != t_len - sizeof(uint32))
        ereport(ERROR,
                (errcode_for_file_access(),
                 errmsg_internal("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
                                 tape, t_len - sizeof(uint32), nread)));
 
    return tuple;
}

References ereport, errcode_for_file_access(), errmsg_internal(), ERROR, hash(), HashAggBatch::input_tape, LogicalTapeRead(), palloc(), and MinimalTupleData::t_len.

Referenced by agg_refill_hash_table().

◆ hashagg_finish_initial_spills()

static void hashagg_finish_initial_spills ( AggState * aggstate )

static

Definition at line 3165 of file nodeAgg.c.

{
    int         setno;
    int         total_npartitions = 0;
 
    if (aggstate->hash_spills != NULL)
    {
        for (setno = 0; setno < aggstate->num_hashes; setno++)
        {
            HashAggSpill *spill = &aggstate->hash_spills[setno];
 
            total_npartitions += spill->npartitions;
            hashagg_spill_finish(aggstate, spill, setno);
        }
 
        /*
         * We're not processing tuples from outer plan any more; only
         * processing batches of spilled tuples. The initial spill structures
         * are no longer needed.
         */
        pfree(aggstate->hash_spills);
        aggstate->hash_spills = NULL;
    }
 
    hash_agg_update_metrics(aggstate, false, total_npartitions);
    aggstate->hash_spill_mode = false;
}

References hash_agg_update_metrics(), AggState::hash_spill_mode, AggState::hash_spills, hashagg_spill_finish(), HashAggSpill::npartitions, AggState::num_hashes, and pfree().

Referenced by agg_fill_hash_table(), and agg_retrieve_direct().

◆ hashagg_recompile_expressions()

static void hashagg_recompile_expressions	(	AggState *	aggstate,
		bool	minslot,
		bool	nullcheck
	)

static

Definition at line 1752 of file nodeAgg.c.

{
    AggStatePerPhase phase;
    int         i = minslot ? 1 : 0;
    int         j = nullcheck ? 1 : 0;
 
    Assert(aggstate->aggstrategy == AGG_HASHED ||
           aggstate->aggstrategy == AGG_MIXED);
 
    if (aggstate->aggstrategy == AGG_HASHED)
        phase = &aggstate->phases[0];
    else                        /* AGG_MIXED */
        phase = &aggstate->phases[1];
 
    if (phase->evaltrans_cache[i][j] == NULL)
    {
        const TupleTableSlotOps *outerops = aggstate->ss.ps.outerops;
        bool        outerfixed = aggstate->ss.ps.outeropsfixed;
        bool        dohash = true;
        bool        dosort = false;
 
        /*
         * If minslot is true, that means we are processing a spilled batch
         * (inside agg_refill_hash_table()), and we must not advance the
         * sorted grouping sets.
         */
        if (aggstate->aggstrategy == AGG_MIXED && !minslot)
            dosort = true;
 
        /* temporarily change the outerops while compiling the expression */
        if (minslot)
        {
            aggstate->ss.ps.outerops = &TTSOpsMinimalTuple;
            aggstate->ss.ps.outeropsfixed = true;
        }
 
        phase->evaltrans_cache[i][j] = ExecBuildAggTrans(aggstate, phase,
                                                         dosort, dohash,
                                                         nullcheck);
 
        /* change back */
        aggstate->ss.ps.outerops = outerops;
        aggstate->ss.ps.outeropsfixed = outerfixed;
    }
 
    phase->evaltrans = phase->evaltrans_cache[i][j];
}

References AGG_HASHED, AGG_MIXED, AggState::aggstrategy, Assert(), AggStatePerPhaseData::evaltrans, AggStatePerPhaseData::evaltrans_cache, ExecBuildAggTrans(), i, j, PlanState::outerops, PlanState::outeropsfixed, AggState::phases, ScanState::ps, AggState::ss, and TTSOpsMinimalTuple.

Referenced by agg_refill_hash_table(), ExecReScanAgg(), and hash_agg_enter_spill_mode().

◆ hashagg_reset_spill_state()

static void hashagg_reset_spill_state ( AggState * aggstate )

static

Definition at line 3239 of file nodeAgg.c.

{
    /* free spills from initial pass */
    if (aggstate->hash_spills != NULL)
    {
        int         setno;
 
        for (setno = 0; setno < aggstate->num_hashes; setno++)
        {
            HashAggSpill *spill = &aggstate->hash_spills[setno];
 
            pfree(spill->ntuples);
            pfree(spill->partitions);
        }
        pfree(aggstate->hash_spills);
        aggstate->hash_spills = NULL;
    }
 
    /* free batches */
    list_free_deep(aggstate->hash_batches);
    aggstate->hash_batches = NIL;
 
    /* close tape set */
    if (aggstate->hash_tapeset != NULL)
    {
        LogicalTapeSetClose(aggstate->hash_tapeset);
        aggstate->hash_tapeset = NULL;
    }
}

References AggState::hash_batches, AggState::hash_spills, AggState::hash_tapeset, list_free_deep(), LogicalTapeSetClose(), NIL, HashAggSpill::ntuples, AggState::num_hashes, HashAggSpill::partitions, and pfree().

Referenced by ExecEndAgg(), and ExecReScanAgg().

◆ hashagg_spill_finish()

static void hashagg_spill_finish	(	AggState *	aggstate,
		HashAggSpill *	spill,
		int	setno
	)

static

Definition at line 3199 of file nodeAgg.c.

{
    int         i;
    int         used_bits = 32 - spill->shift;
 
    if (spill->npartitions == 0)
        return;                 /* didn't spill */
 
    for (i = 0; i < spill->npartitions; i++)
    {
        LogicalTape *tape = spill->partitions[i];
        HashAggBatch *new_batch;
        double      cardinality;
 
        /* if the partition is empty, don't create a new batch of work */
        if (spill->ntuples[i] == 0)
            continue;
 
        cardinality = estimateHyperLogLog(&spill->hll_card[i]);
        freeHyperLogLog(&spill->hll_card[i]);
 
        /* rewinding frees the buffer while not in use */
        LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
 
        new_batch = hashagg_batch_new(tape, setno,
                                      spill->ntuples[i], cardinality,
                                      used_bits);
        aggstate->hash_batches = lappend(aggstate->hash_batches, new_batch);
        aggstate->hash_batches_used++;
    }
 
    pfree(spill->ntuples);
    pfree(spill->hll_card);
    pfree(spill->partitions);
}

References estimateHyperLogLog(), freeHyperLogLog(), AggState::hash_batches, AggState::hash_batches_used, hashagg_batch_new(), HASHAGG_READ_BUFFER_SIZE, HashAggSpill::hll_card, i, lappend(), LogicalTapeRewindForRead(), HashAggSpill::npartitions, HashAggSpill::ntuples, HashAggSpill::partitions, pfree(), and HashAggSpill::shift.

Referenced by agg_refill_hash_table(), and hashagg_finish_initial_spills().

◆ hashagg_spill_init()

static void hashagg_spill_init	(	HashAggSpill *	spill,
		LogicalTapeSet *	tapeset,
		int	used_bits,
		double	input_groups,
		double	hashentrysize
	)

static

Definition at line 2984 of file nodeAgg.c.

{
    int         npartitions;
    int         partition_bits;
 
    npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
                                             used_bits, &partition_bits);
 
#ifdef USE_INJECTION_POINTS
    if (IS_INJECTION_POINT_ATTACHED("hash-aggregate-single-partition"))
    {
        npartitions = 1;
        partition_bits = 0;
        INJECTION_POINT_CACHED("hash-aggregate-single-partition");
    }
#endif
 
    spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
    spill->ntuples = palloc0(sizeof(int64) * npartitions);
    spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
 
    for (int i = 0; i < npartitions; i++)
        spill->partitions[i] = LogicalTapeCreate(tapeset);
 
    spill->shift = 32 - used_bits - partition_bits;
    if (spill->shift < 32)
        spill->mask = (npartitions - 1) << spill->shift;
    else
        spill->mask = 0;
    spill->npartitions = npartitions;
 
    for (int i = 0; i < npartitions; i++)
        initHyperLogLog(&spill->hll_card[i], HASHAGG_HLL_BIT_WIDTH);
}

References for(), hash_choose_num_partitions(), HASHAGG_HLL_BIT_WIDTH, HashAggSpill::hll_card, i, initHyperLogLog(), INJECTION_POINT_CACHED, IS_INJECTION_POINT_ATTACHED, LogicalTapeCreate(), HashAggSpill::mask, HashAggSpill::npartitions, HashAggSpill::ntuples, palloc0(), HashAggSpill::partitions, and HashAggSpill::shift.

Referenced by agg_refill_hash_table(), hash_agg_enter_spill_mode(), and lookup_hash_entries().

◆ hashagg_spill_tuple()

static Size hashagg_spill_tuple	(	AggState *	aggstate,
		HashAggSpill *	spill,
		TupleTableSlot *	inputslot,
		uint32	hash
	)

static

Definition at line 3027 of file nodeAgg.c.

{
    TupleTableSlot *spillslot;
    int         partition;
    MinimalTuple tuple;
    LogicalTape *tape;
    int         total_written = 0;
    bool        shouldFree;
 
    Assert(spill->partitions != NULL);
 
    /* spill only attributes that we actually need */
    if (!aggstate->all_cols_needed)
    {
        spillslot = aggstate->hash_spill_wslot;
        slot_getsomeattrs(inputslot, aggstate->max_colno_needed);
        ExecClearTuple(spillslot);
        for (int i = 0; i < spillslot->tts_tupleDescriptor->natts; i++)
        {
            if (bms_is_member(i + 1, aggstate->colnos_needed))
            {
                spillslot->tts_values[i] = inputslot->tts_values[i];
                spillslot->tts_isnull[i] = inputslot->tts_isnull[i];
            }
            else
                spillslot->tts_isnull[i] = true;
        }
        ExecStoreVirtualTuple(spillslot);
    }
    else
        spillslot = inputslot;
 
    tuple = ExecFetchSlotMinimalTuple(spillslot, &shouldFree);
 
    if (spill->shift < 32)
        partition = (hash & spill->mask) >> spill->shift;
    else
        partition = 0;
 
    spill->ntuples[partition]++;
 
    /*
     * All hash values destined for a given partition have some bits in
     * common, which causes bad HLL cardinality estimates. Hash the hash to
     * get a more uniform distribution.
     */
    addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
 
    tape = spill->partitions[partition];
 
    LogicalTapeWrite(tape, &hash, sizeof(uint32));
    total_written += sizeof(uint32);
 
    LogicalTapeWrite(tape, tuple, tuple->t_len);
    total_written += tuple->t_len;
 
    if (shouldFree)
        pfree(tuple);
 
    return total_written;
}

References addHyperLogLog(), AggState::all_cols_needed, Assert(), bms_is_member(), AggState::colnos_needed, ExecClearTuple(), ExecFetchSlotMinimalTuple(), ExecStoreVirtualTuple(), hash(), hash_bytes_uint32(), AggState::hash_spill_wslot, HashAggSpill::hll_card, i, LogicalTapeWrite(), HashAggSpill::mask, AggState::max_colno_needed, TupleDescData::natts, HashAggSpill::ntuples, HashAggSpill::partitions, pfree(), HashAggSpill::shift, slot_getsomeattrs(), MinimalTupleData::t_len, TupleTableSlot::tts_isnull, TupleTableSlot::tts_tupleDescriptor, and TupleTableSlot::tts_values.

Referenced by agg_refill_hash_table(), and lookup_hash_entries().

◆ initialize_aggregate()

static void initialize_aggregate	(	AggState *	aggstate,
		AggStatePerTrans	pertrans,
		AggStatePerGroup	pergroupstate
	)

static

Definition at line 580 of file nodeAgg.c.

{
    /*
     * Start a fresh sort operation for each DISTINCT/ORDER BY aggregate.
     */
    if (pertrans->aggsortrequired)
    {
        /*
         * In case of rescan, maybe there could be an uncompleted sort
         * operation?  Clean it up if so.
         */
        if (pertrans->sortstates[aggstate->current_set])
            tuplesort_end(pertrans->sortstates[aggstate->current_set]);
 
 
        /*
         * We use a plain Datum sorter when there's a single input column;
         * otherwise sort the full tuple.  (See comments for
         * process_ordered_aggregate_single.)
         */
        if (pertrans->numInputs == 1)
        {
            Form_pg_attribute attr = TupleDescAttr(pertrans->sortdesc, 0);
 
            pertrans->sortstates[aggstate->current_set] =
                tuplesort_begin_datum(attr->atttypid,
                                      pertrans->sortOperators[0],
                                      pertrans->sortCollations[0],
                                      pertrans->sortNullsFirst[0],
                                      work_mem, NULL, TUPLESORT_NONE);
        }
        else
            pertrans->sortstates[aggstate->current_set] =
                tuplesort_begin_heap(pertrans->sortdesc,
                                     pertrans->numSortCols,
                                     pertrans->sortColIdx,
                                     pertrans->sortOperators,
                                     pertrans->sortCollations,
                                     pertrans->sortNullsFirst,
                                     work_mem, NULL, TUPLESORT_NONE);
    }
 
    /*
     * (Re)set transValue to the initial value.
     *
     * Note that when the initial value is pass-by-ref, we must copy it (into
     * the aggcontext) since we will pfree the transValue later.
     */
    if (pertrans->initValueIsNull)
        pergroupstate->transValue = pertrans->initValue;
    else
    {
        MemoryContext oldContext;
 
        oldContext = MemoryContextSwitchTo(aggstate->curaggcontext->ecxt_per_tuple_memory);
        pergroupstate->transValue = datumCopy(pertrans->initValue,
                                              pertrans->transtypeByVal,
                                              pertrans->transtypeLen);
        MemoryContextSwitchTo(oldContext);
    }
    pergroupstate->transValueIsNull = pertrans->initValueIsNull;
 
    /*
     * If the initial value for the transition state doesn't exist in the
     * pg_aggregate table then we will let the first non-NULL value returned
     * from the outer procNode become the initial value. (This is useful for
     * aggregates like max() and min().) The noTransValue flag signals that we
     * still need to do this.
     */
    pergroupstate->noTransValue = pertrans->initValueIsNull;
}

Referenced by initialize_aggregates(), and initialize_hash_entry().

◆ initialize_aggregates()

static void initialize_aggregates	(	AggState *	aggstate,
		AggStatePerGroup *	pergroups,
		int	numReset
	)

static

Definition at line 667 of file nodeAgg.c.

{
    int         transno;
    int         numGroupingSets = Max(aggstate->phase->numsets, 1);
    int         setno = 0;
    int         numTrans = aggstate->numtrans;
    AggStatePerTrans transstates = aggstate->pertrans;
 
    if (numReset == 0)
        numReset = numGroupingSets;
 
    for (setno = 0; setno < numReset; setno++)
    {
        AggStatePerGroup pergroup = pergroups[setno];
 
        select_current_set(aggstate, setno, false);
 
        for (transno = 0; transno < numTrans; transno++)
        {
            AggStatePerTrans pertrans = &transstates[transno];
            AggStatePerGroup pergroupstate = &pergroup[transno];
 
            initialize_aggregate(aggstate, pertrans, pergroupstate);
        }
    }
}

References initialize_aggregate(), Max, AggStatePerPhaseData::numsets, AggState::numtrans, AggState::pertrans, AggState::phase, and select_current_set().

Referenced by agg_retrieve_direct().

◆ initialize_hash_entry()

static void initialize_hash_entry	(	AggState *	aggstate,
		TupleHashTable	hashtable,
		TupleHashEntry	entry
	)

static

Definition at line 2137 of file nodeAgg.c.

{
    AggStatePerGroup pergroup;
    int         transno;
 
    aggstate->hash_ngroups_current++;
    hash_agg_check_limits(aggstate);
 
    /* no need to allocate or initialize per-group state */
    if (aggstate->numtrans == 0)
        return;
 
    pergroup = (AggStatePerGroup) TupleHashEntryGetAdditional(hashtable, entry);
 
    /*
     * Initialize aggregates for new tuple group, lookup_hash_entries()
     * already has selected the relevant grouping set.
     */
    for (transno = 0; transno < aggstate->numtrans; transno++)
    {
        AggStatePerTrans pertrans = &aggstate->pertrans[transno];
        AggStatePerGroup pergroupstate = &pergroup[transno];
 
        initialize_aggregate(aggstate, pertrans, pergroupstate);
    }
}

References hash_agg_check_limits(), AggState::hash_ngroups_current, initialize_aggregate(), AggState::numtrans, AggState::pertrans, and TupleHashEntryGetAdditional().

Referenced by agg_refill_hash_table(), and lookup_hash_entries().

◆ initialize_phase()

static void initialize_phase	(	AggState *	aggstate,
		int	newphase
	)

static

Definition at line 479 of file nodeAgg.c.

{
    Assert(newphase <= 1 || newphase == aggstate->current_phase + 1);
 
    /*
     * Whatever the previous state, we're now done with whatever input
     * tuplesort was in use.
     */
    if (aggstate->sort_in)
    {
        tuplesort_end(aggstate->sort_in);
        aggstate->sort_in = NULL;
    }
 
    if (newphase <= 1)
    {
        /*
         * Discard any existing output tuplesort.
         */
        if (aggstate->sort_out)
        {
            tuplesort_end(aggstate->sort_out);
            aggstate->sort_out = NULL;
        }
    }
    else
    {
        /*
         * The old output tuplesort becomes the new input one, and this is the
         * right time to actually sort it.
         */
        aggstate->sort_in = aggstate->sort_out;
        aggstate->sort_out = NULL;
        Assert(aggstate->sort_in);
        tuplesort_performsort(aggstate->sort_in);
    }
 
    /*
     * If this isn't the last phase, we need to sort appropriately for the
     * next phase in sequence.
     */
    if (newphase > 0 && newphase < aggstate->numphases - 1)
    {
        Sort       *sortnode = aggstate->phases[newphase + 1].sortnode;
        PlanState  *outerNode = outerPlanState(aggstate);
        TupleDesc   tupDesc = ExecGetResultType(outerNode);
 
        aggstate->sort_out = tuplesort_begin_heap(tupDesc,
                                                  sortnode->numCols,
                                                  sortnode->sortColIdx,
                                                  sortnode->sortOperators,
                                                  sortnode->collations,
                                                  sortnode->nullsFirst,
                                                  work_mem,
                                                  NULL, TUPLESORT_NONE);
    }
 
    aggstate->current_phase = newphase;
    aggstate->phase = &aggstate->phases[newphase];
}

References Assert(), AggState::current_phase, ExecGetResultType(), Sort::numCols, outerPlanState, AggState::phase, AggState::phases, AggState::sort_in, AggState::sort_out, AggStatePerPhaseData::sortnode, tuplesort_begin_heap(), tuplesort_end(), TUPLESORT_NONE, tuplesort_performsort(), and work_mem.

Referenced by agg_retrieve_direct(), ExecInitAgg(), and ExecReScanAgg().

◆ lookup_hash_entries()

static void lookup_hash_entries ( AggState * aggstate )

static

Definition at line 2181 of file nodeAgg.c.

{
    AggStatePerGroup *pergroup = aggstate->hash_pergroup;
    TupleTableSlot *outerslot = aggstate->tmpcontext->ecxt_outertuple;
    int         setno;
 
    for (setno = 0; setno < aggstate->num_hashes; setno++)
    {
        AggStatePerHash perhash = &aggstate->perhash[setno];
        TupleHashTable hashtable = perhash->hashtable;
        TupleTableSlot *hashslot = perhash->hashslot;
        TupleHashEntry entry;
        uint32      hash;
        bool        isnew = false;
        bool       *p_isnew;
 
        /* if hash table already spilled, don't create new entries */
        p_isnew = aggstate->hash_spill_mode ? NULL : &isnew;
 
        select_current_set(aggstate, setno, true);
        prepare_hash_slot(perhash,
                          outerslot,
                          hashslot);
 
        entry = LookupTupleHashEntry(hashtable, hashslot,
                                     p_isnew, &hash);
 
        if (entry != NULL)
        {
            if (isnew)
                initialize_hash_entry(aggstate, hashtable, entry);
            pergroup[setno] = TupleHashEntryGetAdditional(hashtable, entry);
        }
        else
        {
            HashAggSpill *spill = &aggstate->hash_spills[setno];
            TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
 
            if (spill->partitions == NULL)
                hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
                                   perhash->aggnode->numGroups,
                                   aggstate->hashentrysize);
 
            hashagg_spill_tuple(aggstate, spill, slot, hash);
            pergroup[setno] = NULL;
        }
    }
}

References AggStatePerHashData::aggnode, ExprContext::ecxt_outertuple, hash(), AggState::hash_pergroup, AggState::hash_spill_mode, AggState::hash_spills, AggState::hash_tapeset, hashagg_spill_init(), hashagg_spill_tuple(), AggState::hashentrysize, AggStatePerHashData::hashslot, AggStatePerHashData::hashtable, initialize_hash_entry(), LookupTupleHashEntry(), AggState::num_hashes, Agg::numGroups, HashAggSpill::partitions, AggState::perhash, prepare_hash_slot(), select_current_set(), AggState::tmpcontext, and TupleHashEntryGetAdditional().

Referenced by agg_fill_hash_table(), and agg_retrieve_direct().

◆ prepare_hash_slot()

static void prepare_hash_slot	(	AggStatePerHash	perhash,
		TupleTableSlot *	inputslot,
		TupleTableSlot *	hashslot
	)

inlinestatic

Definition at line 1203 of file nodeAgg.c.

{
    int         i;
 
    /* transfer just the needed columns into hashslot */
    slot_getsomeattrs(inputslot, perhash->largestGrpColIdx);
    ExecClearTuple(hashslot);
 
    for (i = 0; i < perhash->numhashGrpCols; i++)
    {
        int         varNumber = perhash->hashGrpColIdxInput[i] - 1;
 
        hashslot->tts_values[i] = inputslot->tts_values[varNumber];
        hashslot->tts_isnull[i] = inputslot->tts_isnull[varNumber];
    }
    ExecStoreVirtualTuple(hashslot);
}

References ExecClearTuple(), ExecStoreVirtualTuple(), AggStatePerHashData::hashGrpColIdxInput, i, AggStatePerHashData::largestGrpColIdx, AggStatePerHashData::numhashGrpCols, slot_getsomeattrs(), TupleTableSlot::tts_isnull, and TupleTableSlot::tts_values.

Referenced by agg_refill_hash_table(), and lookup_hash_entries().

◆ prepare_projection_slot()

static void prepare_projection_slot	(	AggState *	aggstate,
		TupleTableSlot *	slot,
		int	currentSet
	)

static

Definition at line 1248 of file nodeAgg.c.

{
    if (aggstate->phase->grouped_cols)
    {
        Bitmapset  *grouped_cols = aggstate->phase->grouped_cols[currentSet];
 
        aggstate->grouped_cols = grouped_cols;
 
        if (TTS_EMPTY(slot))
        {
            /*
             * Force all values to be NULL if working on an empty input tuple
             * (i.e. an empty grouping set for which no input rows were
             * supplied).
             */
            ExecStoreAllNullTuple(slot);
        }
        else if (aggstate->all_grouped_cols)
        {
            ListCell   *lc;
 
            /* all_grouped_cols is arranged in desc order */
            slot_getsomeattrs(slot, linitial_int(aggstate->all_grouped_cols));
 
            foreach(lc, aggstate->all_grouped_cols)
            {
                int         attnum = lfirst_int(lc);
 
                if (!bms_is_member(attnum, grouped_cols))
                    slot->tts_isnull[attnum - 1] = true;
            }
        }
    }
}

References AggState::all_grouped_cols, attnum, bms_is_member(), ExecStoreAllNullTuple(), AggStatePerPhaseData::grouped_cols, AggState::grouped_cols, lfirst_int, linitial_int, AggState::phase, slot_getsomeattrs(), TTS_EMPTY, and TupleTableSlot::tts_isnull.

Referenced by agg_retrieve_direct(), and agg_retrieve_hash_table_in_memory().

◆ process_ordered_aggregate_multi()

static void process_ordered_aggregate_multi	(	AggState *	aggstate,
		AggStatePerTrans	pertrans,
		AggStatePerGroup	pergroupstate
	)

static

Definition at line 948 of file nodeAgg.c.

{
    ExprContext *tmpcontext = aggstate->tmpcontext;
    FunctionCallInfo fcinfo = pertrans->transfn_fcinfo;
    TupleTableSlot *slot1 = pertrans->sortslot;
    TupleTableSlot *slot2 = pertrans->uniqslot;
    int         numTransInputs = pertrans->numTransInputs;
    int         numDistinctCols = pertrans->numDistinctCols;
    Datum       newAbbrevVal = (Datum) 0;
    Datum       oldAbbrevVal = (Datum) 0;
    bool        haveOldValue = false;
    TupleTableSlot *save = aggstate->tmpcontext->ecxt_outertuple;
    int         i;
 
    tuplesort_performsort(pertrans->sortstates[aggstate->current_set]);
 
    ExecClearTuple(slot1);
    if (slot2)
        ExecClearTuple(slot2);
 
    while (tuplesort_gettupleslot(pertrans->sortstates[aggstate->current_set],
                                  true, true, slot1, &newAbbrevVal))
    {
        CHECK_FOR_INTERRUPTS();
 
        tmpcontext->ecxt_outertuple = slot1;
        tmpcontext->ecxt_innertuple = slot2;
 
        if (numDistinctCols == 0 ||
            !haveOldValue ||
            newAbbrevVal != oldAbbrevVal ||
            !ExecQual(pertrans->equalfnMulti, tmpcontext))
        {
            /*
             * Extract the first numTransInputs columns as datums to pass to
             * the transfn.
             */
            slot_getsomeattrs(slot1, numTransInputs);
 
            /* Load values into fcinfo */
            /* Start from 1, since the 0th arg will be the transition value */
            for (i = 0; i < numTransInputs; i++)
            {
                fcinfo->args[i + 1].value = slot1->tts_values[i];
                fcinfo->args[i + 1].isnull = slot1->tts_isnull[i];
            }
 
            advance_transition_function(aggstate, pertrans, pergroupstate);
 
            if (numDistinctCols > 0)
            {
                /* swap the slot pointers to retain the current tuple */
                TupleTableSlot *tmpslot = slot2;
 
                slot2 = slot1;
                slot1 = tmpslot;
                /* avoid ExecQual() calls by reusing abbreviated keys */
                oldAbbrevVal = newAbbrevVal;
                haveOldValue = true;
            }
        }
 
        /* Reset context each time */
        ResetExprContext(tmpcontext);
 
        ExecClearTuple(slot1);
    }
 
    if (slot2)
        ExecClearTuple(slot2);
 
    tuplesort_end(pertrans->sortstates[aggstate->current_set]);
    pertrans->sortstates[aggstate->current_set] = NULL;
 
    /* restore previous slot, potentially in use for grouping sets */
    tmpcontext->ecxt_outertuple = save;
}

References advance_transition_function(), FunctionCallInfoBaseData::args, CHECK_FOR_INTERRUPTS, AggState::current_set, ExprContext::ecxt_innertuple, ExprContext::ecxt_outertuple, AggStatePerTransData::equalfnMulti, ExecClearTuple(), ExecQual(), i, NullableDatum::isnull, AggStatePerTransData::numDistinctCols, AggStatePerTransData::numTransInputs, ResetExprContext, slot_getsomeattrs(), AggStatePerTransData::sortslot, AggStatePerTransData::sortstates, AggState::tmpcontext, AggStatePerTransData::transfn_fcinfo, TupleTableSlot::tts_isnull, TupleTableSlot::tts_values, tuplesort_end(), tuplesort_gettupleslot(), tuplesort_performsort(), AggStatePerTransData::uniqslot, and NullableDatum::value.

Referenced by finalize_aggregates().

◆ process_ordered_aggregate_single()

static void process_ordered_aggregate_single	(	AggState *	aggstate,
		AggStatePerTrans	pertrans,
		AggStatePerGroup	pergroupstate
	)

static

Definition at line 847 of file nodeAgg.c.

{
    Datum       oldVal = (Datum) 0;
    bool        oldIsNull = true;
    bool        haveOldVal = false;
    MemoryContext workcontext = aggstate->tmpcontext->ecxt_per_tuple_memory;
    MemoryContext oldContext;
    bool        isDistinct = (pertrans->numDistinctCols > 0);
    Datum       newAbbrevVal = (Datum) 0;
    Datum       oldAbbrevVal = (Datum) 0;
    FunctionCallInfo fcinfo = pertrans->transfn_fcinfo;
    Datum      *newVal;
    bool       *isNull;
 
    Assert(pertrans->numDistinctCols < 2);
 
    tuplesort_performsort(pertrans->sortstates[aggstate->current_set]);
 
    /* Load the column into argument 1 (arg 0 will be transition value) */
    newVal = &fcinfo->args[1].value;
    isNull = &fcinfo->args[1].isnull;
 
    /*
     * Note: if input type is pass-by-ref, the datums returned by the sort are
     * freshly palloc'd in the per-query context, so we must be careful to
     * pfree them when they are no longer needed.
     */
 
    while (tuplesort_getdatum(pertrans->sortstates[aggstate->current_set],
                              true, false, newVal, isNull, &newAbbrevVal))
    {
        /*
         * Clear and select the working context for evaluation of the equality
         * function and transition function.
         */
        MemoryContextReset(workcontext);
        oldContext = MemoryContextSwitchTo(workcontext);
 
        /*
         * If DISTINCT mode, and not distinct from prior, skip it.
         */
        if (isDistinct &&
            haveOldVal &&
            ((oldIsNull && *isNull) ||
             (!oldIsNull && !*isNull &&
              oldAbbrevVal == newAbbrevVal &&
              DatumGetBool(FunctionCall2Coll(&pertrans->equalfnOne,
                                             pertrans->aggCollation,
                                             oldVal, *newVal)))))
        {
            MemoryContextSwitchTo(oldContext);
            continue;
        }
        else
        {
            advance_transition_function(aggstate, pertrans, pergroupstate);
 
            MemoryContextSwitchTo(oldContext);
 
            /*
             * Forget the old value, if any, and remember the new one for
             * subsequent equality checks.
             */
            if (!pertrans->inputtypeByVal)
            {
                if (!oldIsNull)
                    pfree(DatumGetPointer(oldVal));
                if (!*isNull)
                    oldVal = datumCopy(*newVal, pertrans->inputtypeByVal,
                                       pertrans->inputtypeLen);
            }
            else
                oldVal = *newVal;
            oldAbbrevVal = newAbbrevVal;
            oldIsNull = *isNull;
            haveOldVal = true;
        }
    }
 
    if (!oldIsNull && !pertrans->inputtypeByVal)
        pfree(DatumGetPointer(oldVal));
 
    tuplesort_end(pertrans->sortstates[aggstate->current_set]);
    pertrans->sortstates[aggstate->current_set] = NULL;
}

References advance_transition_function(), AggStatePerTransData::aggCollation, FunctionCallInfoBaseData::args, Assert(), AggState::current_set, datumCopy(), DatumGetBool(), DatumGetPointer(), ExprContext::ecxt_per_tuple_memory, AggStatePerTransData::equalfnOne, FunctionCall2Coll(), AggStatePerTransData::inputtypeByVal, AggStatePerTransData::inputtypeLen, NullableDatum::isnull, MemoryContextReset(), MemoryContextSwitchTo(), AggStatePerTransData::numDistinctCols, pfree(), AggStatePerTransData::sortstates, AggState::tmpcontext, AggStatePerTransData::transfn_fcinfo, tuplesort_end(), tuplesort_getdatum(), tuplesort_performsort(), and NullableDatum::value.

Referenced by finalize_aggregates().

◆ project_aggregates()

static TupleTableSlot * project_aggregates ( AggState * aggstate )

static

Definition at line 1370 of file nodeAgg.c.

{
    ExprContext *econtext = aggstate->ss.ps.ps_ExprContext;
 
    /*
     * Check the qual (HAVING clause); if the group does not match, ignore it.
     */
    if (ExecQual(aggstate->ss.ps.qual, econtext))
    {
        /*
         * Form and return projection tuple using the aggregate results and
         * the representative input tuple.
         */
        return ExecProject(aggstate->ss.ps.ps_ProjInfo);
    }
    else
        InstrCountFiltered1(aggstate, 1);
 
    return NULL;
}

References ExecProject(), ExecQual(), InstrCountFiltered1, ScanState::ps, PlanState::ps_ExprContext, PlanState::ps_ProjInfo, PlanState::qual, and AggState::ss.

Referenced by agg_retrieve_direct(), and agg_retrieve_hash_table_in_memory().

◆ select_current_set()

static void select_current_set	(	AggState *	aggstate,
		int	setno,
		bool	is_hash
	)

static

Definition at line 457 of file nodeAgg.c.

{
    /*
     * When changing this, also adapt ExecAggPlainTransByVal() and
     * ExecAggPlainTransByRef().
     */
    if (is_hash)
        aggstate->curaggcontext = aggstate->hashcontext;
    else
        aggstate->curaggcontext = aggstate->aggcontexts[setno];
 
    aggstate->current_set = setno;
}

References AggState::aggcontexts, AggState::curaggcontext, AggState::current_set, and AggState::hashcontext.

Referenced by agg_fill_hash_table(), agg_refill_hash_table(), agg_retrieve_direct(), agg_retrieve_hash_table_in_memory(), ExecInitAgg(), ExecReScanAgg(), initialize_aggregates(), and lookup_hash_entries().

Data Structures

Macros

Typedefs

Functions

Macro Definition Documentation

◆ CHUNKHDRSZ

◆ HASHAGG_HLL_BIT_WIDTH

◆ HASHAGG_MAX_PARTITIONS

◆ HASHAGG_MIN_PARTITIONS

◆ HASHAGG_PARTITION_FACTOR

◆ HASHAGG_READ_BUFFER_SIZE

◆ HASHAGG_WRITE_BUFFER_SIZE

Typedef Documentation

◆ FindColsContext

◆ HashAggBatch

◆ HashAggSpill

Function Documentation

◆ advance_aggregates()

◆ advance_transition_function()

◆ agg_fill_hash_table()

◆ agg_refill_hash_table()

◆ agg_retrieve_direct()

◆ agg_retrieve_hash_table()

◆ agg_retrieve_hash_table_in_memory()

◆ AggCheckCallContext()

◆ AggGetAggref()

◆ AggGetTempMemoryContext()

◆ AggRegisterCallback()

◆ AggStateIsShared()

◆ build_hash_table()

◆ build_hash_tables()

◆ build_pertrans_for_aggref()

◆ ExecAgg()

◆ ExecAggEstimate()

◆ ExecAggInitializeDSM()

◆ ExecAggInitializeWorker()

◆ ExecAggRetrieveInstrumentation()

◆ ExecEndAgg()

◆ ExecInitAgg()

◆ ExecReScanAgg()

◆ fetch_input_tuple()

◆ finalize_aggregate()

◆ finalize_aggregates()

◆ finalize_partialaggregate()

◆ find_cols()

◆ find_cols_walker()

◆ find_hash_columns()

◆ GetAggInitVal()

◆ hash_agg_check_limits()

◆ hash_agg_enter_spill_mode()

◆ hash_agg_entry_size()

◆ hash_agg_set_limits()

◆ hash_agg_update_metrics()

◆ hash_choose_num_buckets()

◆ hash_choose_num_partitions()

◆ hash_create_memory()

◆ hashagg_batch_new()

◆ hashagg_batch_read()

◆ hashagg_finish_initial_spills()

◆ hashagg_recompile_expressions()

◆ hashagg_reset_spill_state()

◆ hashagg_spill_finish()

◆ hashagg_spill_init()

◆ hashagg_spill_tuple()

◆ initialize_aggregate()

◆ initialize_aggregates()

◆ initialize_hash_entry()

◆ initialize_phase()

◆ lookup_hash_entries()

◆ prepare_hash_slot()

◆ prepare_projection_slot()

◆ process_ordered_aggregate_multi()

◆ process_ordered_aggregate_single()

◆ project_aggregates()

◆ select_current_set()