slab.c File Reference

#include "postgres.h"
#include "lib/ilist.h"
#include "utils/memdebug.h"
#include "utils/memutils.h"
#include "utils/memutils_internal.h"
#include "utils/memutils_memorychunk.h"

Include dependency graph for slab.c:

Go to the source code of this file.

Data Structures
struct	SlabContext
struct	SlabBlock

Macros
#define	Slab_BLOCKHDRSZ   MAXALIGN(sizeof(SlabBlock))
#define	Slab_CONTEXT_HDRSZ(chunksPerBlock)   sizeof(SlabContext)
#define	SLAB_BLOCKLIST_COUNT   3
#define	SLAB_MAXIMUM_EMPTY_BLOCKS   10
#define	Slab_CHUNKHDRSZ   sizeof(MemoryChunk)
#define	SlabChunkGetPointer(chk)    ((void *) (((char *) (chk)) + sizeof(MemoryChunk)))
#define	SlabBlockGetChunk(slab, block, n)
#define	SlabIsValid(set)   ((set) && IsA(set, SlabContext))
#define	SlabBlockIsValid(block)    ((block) && SlabIsValid((block)->slab))

Typedefs
typedef struct SlabContext	SlabContext
typedef struct SlabBlock	SlabBlock

Functions
static int32	SlabBlocklistIndex (SlabContext *slab, int nfree)
static int32	SlabFindNextBlockListIndex (SlabContext *slab)
static MemoryChunk *	SlabGetNextFreeChunk (SlabContext *slab, SlabBlock *block)
MemoryContext	SlabContextCreate (MemoryContext parent, const char *name, Size blockSize, Size chunkSize)
void	SlabReset (MemoryContext context)
void	SlabDelete (MemoryContext context)
static void *	SlabAllocSetupNewChunk (MemoryContext context, SlabBlock *block, MemoryChunk *chunk, Size size)
static pg_noinline void *	SlabAllocFromNewBlock (MemoryContext context, Size size, int flags)
pg_noinline static pg_noreturn void	SlabAllocInvalidSize (MemoryContext context, Size size)
void *	SlabAlloc (MemoryContext context, Size size, int flags)
void	SlabFree (void *pointer)
void *	SlabRealloc (void *pointer, Size size, int flags)
MemoryContext	SlabGetChunkContext (void *pointer)
Size	SlabGetChunkSpace (void *pointer)
bool	SlabIsEmpty (MemoryContext context)
void	SlabStats (MemoryContext context, MemoryStatsPrintFunc printfunc, void *passthru, MemoryContextCounters *totals, bool print_to_stderr)

Macro Definition Documentation

Slab_BLOCKHDRSZ

#define Slab_BLOCKHDRSZ   MAXALIGN(sizeof(SlabBlock))

Definition at line 77 of file slab.c.

SLAB_BLOCKLIST_COUNT

#define SLAB_BLOCKLIST_COUNT   3

Definition at line 95 of file slab.c.

Slab_CHUNKHDRSZ

#define Slab_CHUNKHDRSZ   sizeof(MemoryChunk)

Definition at line 157 of file slab.c.

Slab_CONTEXT_HDRSZ

#define Slab_CONTEXT_HDRSZ

(

chunksPerBlock

)

sizeof(SlabContext)

Definition at line 88 of file slab.c.

SLAB_MAXIMUM_EMPTY_BLOCKS

#define SLAB_MAXIMUM_EMPTY_BLOCKS   10

Definition at line 98 of file slab.c.

SlabBlockGetChunk

#define SlabBlockGetChunk	(		slab,
			block,
			n
	)

Value:

    ((MemoryChunk *) ((char *) (block) + Slab_BLOCKHDRSZ    \
                    + ((n) * (slab)->fullChunkSize)))

Definition at line 165 of file slab.c.

{
    int32       index;
    int32       blocklist_shift = slab->blocklist_shift;
 
    Assert(nfree >= 0 && nfree <= slab->chunksPerBlock);
 
    /*
     * Determine the blocklist index based on the number of free chunks.  We
     * must ensure that 0 free chunks is dedicated to index 0.  Everything
     * else must be >= 1 and < SLAB_BLOCKLIST_COUNT.
     *
     * To make this as efficient as possible, we exploit some two's complement
     * arithmetic where we reverse the sign before bit shifting.  This results
     * in an nfree of 0 using index 0 and anything non-zero staying non-zero.
     * This is exploiting 0 and -0 being the same in two's complement.  When
     * we're done, we just need to flip the sign back over again for a
     * positive index.
     */
    index = -((-nfree) >> blocklist_shift);
 
    if (nfree == 0)
        Assert(index == 0);
    else
        Assert(index >= 1 && index < SLAB_BLOCKLIST_COUNT);
 
    return index;
}
 
/*
 * SlabFindNextBlockListIndex
 *      Search blocklist for blocks which have free chunks and return the
 *      index of the blocklist found containing at least 1 block with free
 *      chunks.  If no block can be found we return 0.
 *
 * Note: We give priority to fuller blocks so that these are filled before
 * emptier blocks.  This is done to increase the chances that mostly-empty
 * blocks will eventually become completely empty so they can be free'd.
 */
static int32
SlabFindNextBlockListIndex(SlabContext *slab)
{
    /* start at 1 as blocklist[0] is for full blocks. */
    for (int i = 1; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        /* return the first found non-empty index */
        if (!dlist_is_empty(&slab->blocklist[i]))
            return i;
    }
 
    /* no blocks with free space */
    return 0;
}
 
/*
 * SlabGetNextFreeChunk
 *      Return the next free chunk in block and update the block to account
 *      for the returned chunk now being used.
 */
static inline MemoryChunk *
SlabGetNextFreeChunk(SlabContext *slab, SlabBlock *block)
{
    MemoryChunk *chunk;
 
    Assert(block->nfree > 0);
 
    if (block->freehead != NULL)
    {
        chunk = block->freehead;
 
        /*
         * Pop the chunk from the linked list of free chunks.  The pointer to
         * the next free chunk is stored in the chunk itself.
         */
        VALGRIND_MAKE_MEM_DEFINED(SlabChunkGetPointer(chunk), sizeof(MemoryChunk *));
        block->freehead = *(MemoryChunk **) SlabChunkGetPointer(chunk);
 
        /* check nothing stomped on the free chunk's memory */
        Assert(block->freehead == NULL ||
               (block->freehead >= SlabBlockGetChunk(slab, block, 0) &&
                block->freehead <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1) &&
                SlabChunkMod(slab, block, block->freehead) == 0));
    }
    else
    {
        Assert(block->nunused > 0);
 
        chunk = block->unused;
        block->unused = (MemoryChunk *) (((char *) block->unused) + slab->fullChunkSize);
        block->nunused--;
    }
 
    block->nfree--;
 
    return chunk;
}
 
/*
 * SlabContextCreate
 *      Create a new Slab context.
 *
 * parent: parent context, or NULL if top-level context
 * name: name of context (must be statically allocated)
 * blockSize: allocation block size
 * chunkSize: allocation chunk size
 *
 * The Slab_CHUNKHDRSZ + MAXALIGN(chunkSize + 1) may not exceed
 * MEMORYCHUNK_MAX_VALUE.
 * 'blockSize' may not exceed MEMORYCHUNK_MAX_BLOCKOFFSET.
 */
MemoryContext
SlabContextCreate(MemoryContext parent,
                  const char *name,
                  Size blockSize,
                  Size chunkSize)
{
    int         chunksPerBlock;
    Size        fullChunkSize;
    SlabContext *slab;
    int         i;
 
    /* ensure MemoryChunk's size is properly maxaligned */
    StaticAssertDecl(Slab_CHUNKHDRSZ == MAXALIGN(Slab_CHUNKHDRSZ),
                     "sizeof(MemoryChunk) is not maxaligned");
    Assert(blockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
 
    /*
     * Ensure there's enough space to store the pointer to the next free chunk
     * in the memory of the (otherwise) unused allocation.
     */
    if (chunkSize < sizeof(MemoryChunk *))
        chunkSize = sizeof(MemoryChunk *);
 
    /* length of the maxaligned chunk including the chunk header  */
#ifdef MEMORY_CONTEXT_CHECKING
    /* ensure there's always space for the sentinel byte */
    fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize + 1);
#else
    fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize);
#endif
 
    Assert(fullChunkSize <= MEMORYCHUNK_MAX_VALUE);
 
    /* compute the number of chunks that will fit on each block */
    chunksPerBlock = (blockSize - Slab_BLOCKHDRSZ) / fullChunkSize;
 
    /* Make sure the block can store at least one chunk. */
    if (chunksPerBlock == 0)
        elog(ERROR, "block size %zu for slab is too small for %zu-byte chunks",
             blockSize, chunkSize);
 
 
 
    slab = (SlabContext *) malloc(Slab_CONTEXT_HDRSZ(chunksPerBlock));
    if (slab == NULL)
    {
        MemoryContextStats(TopMemoryContext);
        ereport(ERROR,
                (errcode(ERRCODE_OUT_OF_MEMORY),
                 errmsg("out of memory"),
                 errdetail("Failed while creating memory context \"%s\".",
                           name)));
    }
 
    /*
     * Avoid writing code that can fail between here and MemoryContextCreate;
     * we'd leak the header if we ereport in this stretch.
     */
 
    /* See comments about Valgrind interactions in aset.c */
    VALGRIND_CREATE_MEMPOOL(slab, 0, false);
    /* This vchunk covers the SlabContext only */
    VALGRIND_MEMPOOL_ALLOC(slab, slab, sizeof(SlabContext));
 
    /* Fill in SlabContext-specific header fields */
    slab->chunkSize = (uint32) chunkSize;
    slab->fullChunkSize = (uint32) fullChunkSize;
    slab->blockSize = (uint32) blockSize;
    slab->chunksPerBlock = chunksPerBlock;
    slab->curBlocklistIndex = 0;
 
    /*
     * Compute a shift that guarantees that shifting chunksPerBlock with it is
     * < SLAB_BLOCKLIST_COUNT - 1.  The reason that we subtract 1 from
     * SLAB_BLOCKLIST_COUNT in this calculation is that we reserve the 0th
     * blocklist element for blocks which have no free chunks.
     *
     * We calculate the number of bits to shift by rather than a divisor to
     * divide by as performing division each time we need to find the
     * blocklist index would be much slower.
     */
    slab->blocklist_shift = 0;
    while ((slab->chunksPerBlock >> slab->blocklist_shift) >= (SLAB_BLOCKLIST_COUNT - 1))
        slab->blocklist_shift++;
 
    /* initialize the list to store empty blocks to be reused */
    dclist_init(&slab->emptyblocks);
 
    /* initialize each blocklist slot */
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
        dlist_init(&slab->blocklist[i]);
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* set the isChunkFree pointer right after the end of the context */
    slab->isChunkFree = (bool *) ((char *) slab + sizeof(SlabContext));
#endif
 
    /* Finally, do the type-independent part of context creation */
    MemoryContextCreate((MemoryContext) slab,
                        T_SlabContext,
                        MCTX_SLAB_ID,
                        parent,
                        name);
 
    return (MemoryContext) slab;
}
 
/*
 * SlabReset
 *      Frees all memory which is allocated in the given set.
 *
 * The code simply frees all the blocks in the context - we don't keep any
 * keeper blocks or anything like that.
 */
void
SlabReset(MemoryContext context)
{
    SlabContext *slab = (SlabContext *) context;
    dlist_mutable_iter miter;
    int         i;
 
    Assert(SlabIsValid(slab));
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* Check for corruption and leaks before freeing */
    SlabCheck(context);
#endif
 
    /* release any retained empty blocks */
    dclist_foreach_modify(miter, &slab->emptyblocks)
    {
        SlabBlock  *block = dlist_container(SlabBlock, node, miter.cur);
 
        dclist_delete_from(&slab->emptyblocks, miter.cur);
 
#ifdef CLOBBER_FREED_MEMORY
        wipe_mem(block, slab->blockSize);
#endif
 
        /* As in aset.c, free block-header vchunks explicitly */
        VALGRIND_MEMPOOL_FREE(slab, block);
 
        free(block);
        context->mem_allocated -= slab->blockSize;
    }
 
    /* walk over blocklist and free the blocks */
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        dlist_foreach_modify(miter, &slab->blocklist[i])
        {
            SlabBlock  *block = dlist_container(SlabBlock, node, miter.cur);
 
            dlist_delete(miter.cur);
 
#ifdef CLOBBER_FREED_MEMORY
            wipe_mem(block, slab->blockSize);
#endif
 
            /* As in aset.c, free block-header vchunks explicitly */
            VALGRIND_MEMPOOL_FREE(slab, block);
 
            free(block);
            context->mem_allocated -= slab->blockSize;
        }
    }
 
    /*
     * Instruct Valgrind to throw away all the vchunks associated with this
     * context, except for the one covering the SlabContext.  This gets rid of
     * the vchunks for whatever user data is getting discarded by the context
     * reset.
     */
    VALGRIND_MEMPOOL_TRIM(slab, slab, sizeof(SlabContext));
 
    slab->curBlocklistIndex = 0;
 
    Assert(context->mem_allocated == 0);
}
 
/*
 * SlabDelete
 *      Free all memory which is allocated in the given context.
 */
void
SlabDelete(MemoryContext context)
{
    /* Reset to release all the SlabBlocks */
    SlabReset(context);
 
    /* Destroy the vpool -- see notes in aset.c */
    VALGRIND_DESTROY_MEMPOOL(context);
 
    /* And free the context header */
    free(context);
}
 
/*
 * Small helper for allocating a new chunk from a chunk, to avoid duplicating
 * the code between SlabAlloc() and SlabAllocFromNewBlock().
 */
static inline void *
SlabAllocSetupNewChunk(MemoryContext context, SlabBlock *block,
                       MemoryChunk *chunk, Size size)
{
    SlabContext *slab = (SlabContext *) context;
 
    /*
     * Check that the chunk pointer is actually somewhere on the block and is
     * aligned as expected.
     */
    Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
    Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
    Assert(SlabChunkMod(slab, block, chunk) == 0);
 
    /* Prepare to initialize the chunk header. */
    VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
 
    MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize), MCTX_SLAB_ID);
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* slab mark to catch clobber of "unused" space */
    Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
    set_sentinel(MemoryChunkGetPointer(chunk), size);
    VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) + Slab_CHUNKHDRSZ +
                               slab->chunkSize,
                               slab->fullChunkSize -
                               (slab->chunkSize + Slab_CHUNKHDRSZ));
#endif
 
#ifdef RANDOMIZE_ALLOCATED_MEMORY
    /* fill the allocated space with junk */
    randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
#endif
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    return MemoryChunkGetPointer(chunk);
}
 
pg_noinline
static void *
SlabAllocFromNewBlock(MemoryContext context, Size size, int flags)
{
    SlabContext *slab = (SlabContext *) context;
    SlabBlock  *block;
    MemoryChunk *chunk;
    dlist_head *blocklist;
    int         blocklist_idx;
 
    /* to save allocating a new one, first check the empty blocks list */
    if (dclist_count(&slab->emptyblocks) > 0)
    {
        dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
 
        block = dlist_container(SlabBlock, node, node);
 
        /*
         * SlabFree() should have left this block in a valid state with all
         * chunks free.  Ensure that's the case.
         */
        Assert(block->nfree == slab->chunksPerBlock);
 
        /* fetch the next chunk from this block */
        chunk = SlabGetNextFreeChunk(slab, block);
    }
    else
    {
        block = (SlabBlock *) malloc(slab->blockSize);
 
        if (unlikely(block == NULL))
            return MemoryContextAllocationFailure(context, size, flags);
 
        /* Make a vchunk covering the new block's header */
        VALGRIND_MEMPOOL_ALLOC(slab, block, Slab_BLOCKHDRSZ);
 
        block->slab = slab;
        context->mem_allocated += slab->blockSize;
 
        /* use the first chunk in the new block */
        chunk = SlabBlockGetChunk(slab, block, 0);
 
        block->nfree = slab->chunksPerBlock - 1;
        block->unused = SlabBlockGetChunk(slab, block, 1);
        block->freehead = NULL;
        block->nunused = slab->chunksPerBlock - 1;
    }
 
    /* find the blocklist element for storing blocks with 1 used chunk */
    blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
    blocklist = &slab->blocklist[blocklist_idx];
 
    /* this better be empty.  We just added a block thinking it was */
    Assert(dlist_is_empty(blocklist));
 
    dlist_push_head(blocklist, &block->node);
 
    slab->curBlocklistIndex = blocklist_idx;
 
    return SlabAllocSetupNewChunk(context, block, chunk, size);
}
 
/*
 * SlabAllocInvalidSize
 *      Handle raising an ERROR for an invalid size request.  We don't do this
 *      in slab alloc as calling the elog functions would force the compiler
 *      to setup the stack frame in SlabAlloc.  For performance reasons, we
 *      want to avoid that.
 */
pg_noinline
pg_noreturn
static void
SlabAllocInvalidSize(MemoryContext context, Size size)
{
    SlabContext *slab = (SlabContext *) context;
 
    elog(ERROR, "unexpected alloc chunk size %zu (expected %u)", size,
         slab->chunkSize);
}
 
/*
 * SlabAlloc
 *      Returns a pointer to a newly allocated memory chunk or raises an ERROR
 *      on allocation failure, or returns NULL when flags contains
 *      MCXT_ALLOC_NO_OOM.  'size' must be the same size as was specified
 *      during SlabContextCreate().
 *
 * This function should only contain the most common code paths.  Everything
 * else should be in pg_noinline helper functions, thus avoiding the overhead
 * of creating a stack frame for the common cases.  Allocating memory is often
 * a bottleneck in many workloads, so avoiding stack frame setup is
 * worthwhile.  Helper functions should always directly return the newly
 * allocated memory so that we can just return that address directly as a tail
 * call.
 */
void *
SlabAlloc(MemoryContext context, Size size, int flags)
{
    SlabContext *slab = (SlabContext *) context;
    SlabBlock  *block;
    MemoryChunk *chunk;
 
    Assert(SlabIsValid(slab));
 
    /* sanity check that this is pointing to a valid blocklist */
    Assert(slab->curBlocklistIndex >= 0);
    Assert(slab->curBlocklistIndex <= SlabBlocklistIndex(slab, slab->chunksPerBlock));
 
    /*
     * Make sure we only allow correct request size.  This doubles as the
     * MemoryContextCheckSize check.
     */
    if (unlikely(size != slab->chunkSize))
        SlabAllocInvalidSize(context, size);
 
    if (unlikely(slab->curBlocklistIndex == 0))
    {
        /*
         * Handle the case when there are no partially filled blocks
         * available.  This happens either when the last allocation took the
         * last chunk in the block, or when SlabFree() free'd the final block.
         */
        return SlabAllocFromNewBlock(context, size, flags);
    }
    else
    {
        dlist_head *blocklist = &slab->blocklist[slab->curBlocklistIndex];
        int         new_blocklist_idx;
 
        Assert(!dlist_is_empty(blocklist));
 
        /* grab the block from the blocklist */
        block = dlist_head_element(SlabBlock, node, blocklist);
 
        /* make sure we actually got a valid block, with matching nfree */
        Assert(block != NULL);
        Assert(slab->curBlocklistIndex == SlabBlocklistIndex(slab, block->nfree));
        Assert(block->nfree > 0);
 
        /* fetch the next chunk from this block */
        chunk = SlabGetNextFreeChunk(slab, block);
 
        /* get the new blocklist index based on the new free chunk count */
        new_blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
 
        /*
         * Handle the case where the blocklist index changes.  This also deals
         * with blocks becoming full as only full blocks go at index 0.
         */
        if (unlikely(slab->curBlocklistIndex != new_blocklist_idx))
        {
            dlist_delete_from(blocklist, &block->node);
            dlist_push_head(&slab->blocklist[new_blocklist_idx], &block->node);
 
            if (dlist_is_empty(blocklist))
                slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
        }
    }
 
    return SlabAllocSetupNewChunk(context, block, chunk, size);
}
 
/*
 * SlabFree
 *      Frees allocated memory; memory is removed from the slab.
 */
void
SlabFree(void *pointer)
{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
    SlabContext *slab;
    int         curBlocklistIdx;
    int         newBlocklistIdx;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /*
     * For speed reasons we just Assert that the referenced block is good.
     * Future field experience may show that this Assert had better become a
     * regular runtime test-and-elog check.
     */
    Assert(SlabBlockIsValid(block));
    slab = block->slab;
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* Test for someone scribbling on unused space in chunk */
    Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
    if (!sentinel_ok(pointer, slab->chunkSize))
        elog(WARNING, "detected write past chunk end in %s %p",
             slab->header.name, chunk);
#endif
 
    /* push this chunk onto the head of the block's free list */
    *(MemoryChunk **) pointer = block->freehead;
    block->freehead = chunk;
 
    block->nfree++;
 
    Assert(block->nfree > 0);
    Assert(block->nfree <= slab->chunksPerBlock);
 
#ifdef CLOBBER_FREED_MEMORY
    /* don't wipe the free list MemoryChunk pointer stored in the chunk */
    wipe_mem((char *) pointer + sizeof(MemoryChunk *),
             slab->chunkSize - sizeof(MemoryChunk *));
#endif
 
    curBlocklistIdx = SlabBlocklistIndex(slab, block->nfree - 1);
    newBlocklistIdx = SlabBlocklistIndex(slab, block->nfree);
 
    /*
     * Check if the block needs to be moved to another element on the
     * blocklist based on it now having 1 more free chunk.
     */
    if (unlikely(curBlocklistIdx != newBlocklistIdx))
    {
        /* do the move */
        dlist_delete_from(&slab->blocklist[curBlocklistIdx], &block->node);
        dlist_push_head(&slab->blocklist[newBlocklistIdx], &block->node);
 
        /*
         * The blocklist[curBlocklistIdx] may now be empty or we may now be
         * able to use a lower-element blocklist.  We'll need to redetermine
         * what the slab->curBlocklistIndex is if the current blocklist was
         * changed or if a lower element one was changed.  We must ensure we
         * use the list with the fullest block(s).
         */
        if (slab->curBlocklistIndex >= curBlocklistIdx)
        {
            slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
 
            /*
             * We know there must be a block with at least 1 unused chunk as
             * we just pfree'd one.  Ensure curBlocklistIndex reflects this.
             */
            Assert(slab->curBlocklistIndex > 0);
        }
    }
 
    /* Handle when a block becomes completely empty */
    if (unlikely(block->nfree == slab->chunksPerBlock))
    {
        /* remove the block */
        dlist_delete_from(&slab->blocklist[newBlocklistIdx], &block->node);
 
        /*
         * To avoid thrashing malloc/free, we keep a list of empty blocks that
         * we can reuse again instead of having to malloc a new one.
         */
        if (dclist_count(&slab->emptyblocks) < SLAB_MAXIMUM_EMPTY_BLOCKS)
            dclist_push_head(&slab->emptyblocks, &block->node);
        else
        {
            /*
             * When we have enough empty blocks stored already, we actually
             * free the block.
             */
#ifdef CLOBBER_FREED_MEMORY
            wipe_mem(block, slab->blockSize);
#endif
 
            /* As in aset.c, free block-header vchunks explicitly */
            VALGRIND_MEMPOOL_FREE(slab, block);
 
            free(block);
            slab->header.mem_allocated -= slab->blockSize;
        }
 
        /*
         * Check if we need to reset the blocklist index.  This is required
         * when the blocklist this block is on has become completely empty.
         */
        if (slab->curBlocklistIndex == newBlocklistIdx &&
            dlist_is_empty(&slab->blocklist[newBlocklistIdx]))
            slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
    }
}
 
/*
 * SlabRealloc
 *      Change the allocated size of a chunk.
 *
 * As Slab is designed for allocating equally-sized chunks of memory, it can't
 * do an actual chunk size change.  We try to be gentle and allow calls with
 * exactly the same size, as in that case we can simply return the same
 * chunk.  When the size differs, we throw an error.
 *
 * We could also allow requests with size < chunkSize.  That however seems
 * rather pointless - Slab is meant for chunks of constant size, and moreover
 * realloc is usually used to enlarge the chunk.
 */
void *
SlabRealloc(void *pointer, Size size, int flags)
{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
    SlabContext *slab;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    /*
     * Try to verify that we have a sane block pointer: the block header
     * should reference a slab context.  (We use a test-and-elog, not just
     * Assert, because it seems highly likely that we're here in error in the
     * first place.)
     */
    if (!SlabBlockIsValid(block))
        elog(ERROR, "could not find block containing chunk %p", chunk);
    slab = block->slab;
 
    /* can't do actual realloc with slab, but let's try to be gentle */
    if (size == slab->chunkSize)
        return pointer;
 
    elog(ERROR, "slab allocator does not support realloc()");
    return NULL;                /* keep compiler quiet */
}
 
/*
 * SlabGetChunkContext
 *      Return the MemoryContext that 'pointer' belongs to.
 */
MemoryContext
SlabGetChunkContext(void *pointer)
{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    Assert(SlabBlockIsValid(block));
 
    return &block->slab->header;
}
 
/*
 * SlabGetChunkSpace
 *      Given a currently-allocated chunk, determine the total space
 *      it occupies (including all memory-allocation overhead).
 */
Size
SlabGetChunkSpace(void *pointer)
{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
    SlabContext *slab;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    Assert(SlabBlockIsValid(block));
    slab = block->slab;
 
    return slab->fullChunkSize;
}
 
/*
 * SlabIsEmpty
 *      Is the slab empty of any allocated space?
 */
bool
SlabIsEmpty(MemoryContext context)
{
    Assert(SlabIsValid((SlabContext *) context));
 
    return (context->mem_allocated == 0);
}
 
/*
 * SlabStats
 *      Compute stats about memory consumption of a Slab context.
 *
 * printfunc: if not NULL, pass a human-readable stats string to this.
 * passthru: pass this pointer through to printfunc.
 * totals: if not NULL, add stats about this context into *totals.
 * print_to_stderr: print stats to stderr if true, elog otherwise.
 */
void
SlabStats(MemoryContext context,
          MemoryStatsPrintFunc printfunc, void *passthru,
          MemoryContextCounters *totals,
          bool print_to_stderr)
{
    SlabContext *slab = (SlabContext *) context;
    Size        nblocks = 0;
    Size        freechunks = 0;
    Size        totalspace;
    Size        freespace = 0;
    int         i;
 
    Assert(SlabIsValid(slab));
 
    /* Include context header in totalspace */
    totalspace = Slab_CONTEXT_HDRSZ(slab->chunksPerBlock);
 
    /* Add the space consumed by blocks in the emptyblocks list */
    totalspace += dclist_count(&slab->emptyblocks) * slab->blockSize;
 
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        dlist_iter  iter;
 
        dlist_foreach(iter, &slab->blocklist[i])
        {
            SlabBlock  *block = dlist_container(SlabBlock, node, iter.cur);
 
            nblocks++;
            totalspace += slab->blockSize;
            freespace += slab->fullChunkSize * block->nfree;
            freechunks += block->nfree;
        }
    }
 
    if (printfunc)
    {
        char        stats_string[200];
 
        /* XXX should we include free chunks on empty blocks? */
        snprintf(stats_string, sizeof(stats_string),
                 "%zu total in %zu blocks; %u empty blocks; %zu free (%zu chunks); %zu used",
                 totalspace, nblocks, dclist_count(&slab->emptyblocks),
                 freespace, freechunks, totalspace - freespace);
        printfunc(context, passthru, stats_string, print_to_stderr);
    }
 
    if (totals)
    {
        totals->nblocks += nblocks;
        totals->freechunks += freechunks;
        totals->totalspace += totalspace;
        totals->freespace += freespace;
    }
}
 
 
#ifdef MEMORY_CONTEXT_CHECKING
 
/*
 * SlabCheck
 *      Walk through all blocks looking for inconsistencies.
 *
 * NOTE: report errors as WARNING, *not* ERROR or FATAL.  Otherwise you'll
 * find yourself in an infinite loop when trouble occurs, because this
 * routine will be entered again when elog cleanup tries to release memory!
 */
void
SlabCheck(MemoryContext context)
{
    SlabContext *slab = (SlabContext *) context;
    int         i;
    int         nblocks = 0;
    const char *name = slab->header.name;
    dlist_iter  iter;
 
    Assert(SlabIsValid(slab));
    Assert(slab->chunksPerBlock > 0);
 
    /*
     * Have a look at the empty blocks.  These should have all their chunks
     * marked as free.  Ensure that's the case.
     */
    dclist_foreach(iter, &slab->emptyblocks)
    {
        SlabBlock  *block = dlist_container(SlabBlock, node, iter.cur);
 
        if (block->nfree != slab->chunksPerBlock)
            elog(WARNING, "problem in slab %s: empty block %p should have %d free chunks but has %d chunks free",
                 name, block, slab->chunksPerBlock, block->nfree);
    }
 
    /* walk the non-empty block lists */
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        int         j,
                    nfree;
 
        /* walk all blocks on this blocklist */
        dlist_foreach(iter, &slab->blocklist[i])
        {
            SlabBlock  *block = dlist_container(SlabBlock, node, iter.cur);
            MemoryChunk *cur_chunk;
 
            /*
             * Make sure the number of free chunks (in the block header)
             * matches the position in the blocklist.
             */
            if (SlabBlocklistIndex(slab, block->nfree) != i)
                elog(WARNING, "problem in slab %s: block %p is on blocklist %d but should be on blocklist %d",
                     name, block, i, SlabBlocklistIndex(slab, block->nfree));
 
            /* make sure the block is not empty */
            if (block->nfree >= slab->chunksPerBlock)
                elog(WARNING, "problem in slab %s: empty block %p incorrectly stored on blocklist element %d",
                     name, block, i);
 
            /* make sure the slab pointer correctly points to this context */
            if (block->slab != slab)
                elog(WARNING, "problem in slab %s: bogus slab link in block %p",
                     name, block);
 
            /* reset the array of free chunks for this block */
            memset(slab->isChunkFree, 0, (slab->chunksPerBlock * sizeof(bool)));
            nfree = 0;
 
            /* walk through the block's free list chunks */
            cur_chunk = block->freehead;
            while (cur_chunk != NULL)
            {
                int         chunkidx = SlabChunkIndex(slab, block, cur_chunk);
 
                /*
                 * Ensure the free list link points to something on the block
                 * at an address aligned according to the full chunk size.
                 */
                if (cur_chunk < SlabBlockGetChunk(slab, block, 0) ||
                    cur_chunk > SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1) ||
                    SlabChunkMod(slab, block, cur_chunk) != 0)
                    elog(WARNING, "problem in slab %s: bogus free list link %p in block %p",
                         name, cur_chunk, block);
 
                /* count the chunk and mark it free on the free chunk array */
                nfree++;
                slab->isChunkFree[chunkidx] = true;
 
                /* read pointer of the next free chunk */
                VALGRIND_MAKE_MEM_DEFINED(MemoryChunkGetPointer(cur_chunk), sizeof(MemoryChunk *));
                cur_chunk = *(MemoryChunk **) SlabChunkGetPointer(cur_chunk);
            }
 
            /* check that the unused pointer matches what nunused claims */
            if (SlabBlockGetChunk(slab, block, slab->chunksPerBlock - block->nunused) !=
                block->unused)
                elog(WARNING, "problem in slab %s: mismatch detected between nunused chunks and unused pointer in block %p",
                     name, block);
 
            /*
             * count the remaining free chunks that have yet to make it onto
             * the block's free list.
             */
            cur_chunk = block->unused;
            for (j = 0; j < block->nunused; j++)
            {
                int         chunkidx = SlabChunkIndex(slab, block, cur_chunk);
 
 
                /* count the chunk as free and mark it as so in the array */
                nfree++;
                if (chunkidx < slab->chunksPerBlock)
                    slab->isChunkFree[chunkidx] = true;
 
                /* move forward 1 chunk */
                cur_chunk = (MemoryChunk *) (((char *) cur_chunk) + slab->fullChunkSize);
            }
 
            for (j = 0; j < slab->chunksPerBlock; j++)
            {
                if (!slab->isChunkFree[j])
                {
                    MemoryChunk *chunk = SlabBlockGetChunk(slab, block, j);
                    SlabBlock  *chunkblock;
 
                    /* Allow access to the chunk header. */
                    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
                    chunkblock = (SlabBlock *) MemoryChunkGetBlock(chunk);
 
                    /* Disallow access to the chunk header. */
                    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
                    /*
                     * check the chunk's blockoffset correctly points back to
                     * the block
                     */
                    if (chunkblock != block)
                        elog(WARNING, "problem in slab %s: bogus block link in block %p, chunk %p",
                             name, block, chunk);
 
                    /* check the sentinel byte is intact */
                    Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
                    if (!sentinel_ok(chunk, Slab_CHUNKHDRSZ + slab->chunkSize))
                        elog(WARNING, "problem in slab %s: detected write past chunk end in block %p, chunk %p",
                             name, block, chunk);
                }
            }
 
            /*
             * Make sure we got the expected number of free chunks (as tracked
             * in the block header).
             */
            if (nfree != block->nfree)
                elog(WARNING, "problem in slab %s: nfree in block %p is %d but %d chunk were found as free",
                     name, block, block->nfree, nfree);
 
            nblocks++;
        }
    }
 
    /* the stored empty blocks are tracked in mem_allocated too */
    nblocks += dclist_count(&slab->emptyblocks);
 
    Assert(nblocks * slab->blockSize == context->mem_allocated);
}
 
#endif                          /* MEMORY_CONTEXT_CHECKING */

SlabBlockIsValid

#define SlabBlockIsValid

(

block

)

((block) && SlabIsValid((block)->slab))

Definition at line 202 of file slab.c.

SlabChunkGetPointer

#define SlabChunkGetPointer

(

chk

)

((void *) (((char *) (chk)) + sizeof(MemoryChunk)))

Definition at line 158 of file slab.c.

SlabIsValid

#define SlabIsValid

(

set

)

((set) && IsA(set, SlabContext))

Definition at line 196 of file slab.c.

Typedef Documentation

SlabBlock

typedef struct SlabBlock SlabBlock

SlabContext

typedef struct SlabContext SlabContext

Function Documentation

SlabAlloc()

void * SlabAlloc	(	MemoryContext	context,
		Size	size,
		int	flags
	)

Definition at line 658 of file slab.c.

{
    SlabContext *slab = (SlabContext *) context;
    SlabBlock  *block;
    MemoryChunk *chunk;
 
    Assert(SlabIsValid(slab));
 
    /* sanity check that this is pointing to a valid blocklist */
    Assert(slab->curBlocklistIndex >= 0);
    Assert(slab->curBlocklistIndex <= SlabBlocklistIndex(slab, slab->chunksPerBlock));
 
    /*
     * Make sure we only allow correct request size.  This doubles as the
     * MemoryContextCheckSize check.
     */
    if (unlikely(size != slab->chunkSize))
        SlabAllocInvalidSize(context, size);
 
    if (unlikely(slab->curBlocklistIndex == 0))
    {
        /*
         * Handle the case when there are no partially filled blocks
         * available.  This happens either when the last allocation took the
         * last chunk in the block, or when SlabFree() free'd the final block.
         */
        return SlabAllocFromNewBlock(context, size, flags);
    }
    else
    {
        dlist_head *blocklist = &slab->blocklist[slab->curBlocklistIndex];
        int         new_blocklist_idx;
 
        Assert(!dlist_is_empty(blocklist));
 
        /* grab the block from the blocklist */
        block = dlist_head_element(SlabBlock, node, blocklist);
 
        /* make sure we actually got a valid block, with matching nfree */
        Assert(block != NULL);
        Assert(slab->curBlocklistIndex == SlabBlocklistIndex(slab, block->nfree));
        Assert(block->nfree > 0);
 
        /* fetch the next chunk from this block */
        chunk = SlabGetNextFreeChunk(slab, block);
 
        /* get the new blocklist index based on the new free chunk count */
        new_blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
 
        /*
         * Handle the case where the blocklist index changes.  This also deals
         * with blocks becoming full as only full blocks go at index 0.
         */
        if (unlikely(slab->curBlocklistIndex != new_blocklist_idx))
        {
            dlist_delete_from(blocklist, &block->node);
            dlist_push_head(&slab->blocklist[new_blocklist_idx], &block->node);
 
            if (dlist_is_empty(blocklist))
                slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
        }
    }
 
    return SlabAllocSetupNewChunk(context, block, chunk, size);
}

References Assert, SlabContext::blocklist, SlabContext::chunkSize, SlabContext::chunksPerBlock, SlabContext::curBlocklistIndex, dlist_delete_from(), dlist_head_element, dlist_is_empty(), dlist_push_head(), fb(), SlabBlock::nfree, SlabBlock::node, SlabAllocFromNewBlock(), SlabAllocInvalidSize(), SlabAllocSetupNewChunk(), SlabBlocklistIndex(), SlabFindNextBlockListIndex(), SlabGetNextFreeChunk(), SlabIsValid, and unlikely.

SlabAllocFromNewBlock()

static pg_noinline void * SlabAllocFromNewBlock ( MemoryContext  context, Size  size, int  flags  )

static

Definition at line 564 of file slab.c.

{
    SlabContext *slab = (SlabContext *) context;
    SlabBlock  *block;
    MemoryChunk *chunk;
    dlist_head *blocklist;
    int         blocklist_idx;
 
    /* to save allocating a new one, first check the empty blocks list */
    if (dclist_count(&slab->emptyblocks) > 0)
    {
        dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
 
        block = dlist_container(SlabBlock, node, node);
 
        /*
         * SlabFree() should have left this block in a valid state with all
         * chunks free.  Ensure that's the case.
         */
        Assert(block->nfree == slab->chunksPerBlock);
 
        /* fetch the next chunk from this block */
        chunk = SlabGetNextFreeChunk(slab, block);
    }
    else
    {
        block = (SlabBlock *) malloc(slab->blockSize);
 
        if (unlikely(block == NULL))
            return MemoryContextAllocationFailure(context, size, flags);
 
        /* Make a vchunk covering the new block's header */
        VALGRIND_MEMPOOL_ALLOC(slab, block, Slab_BLOCKHDRSZ);
 
        block->slab = slab;
        context->mem_allocated += slab->blockSize;
 
        /* use the first chunk in the new block */
        chunk = SlabBlockGetChunk(slab, block, 0);
 
        block->nfree = slab->chunksPerBlock - 1;
        block->unused = SlabBlockGetChunk(slab, block, 1);
        block->freehead = NULL;
        block->nunused = slab->chunksPerBlock - 1;
    }
 
    /* find the blocklist element for storing blocks with 1 used chunk */
    blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
    blocklist = &slab->blocklist[blocklist_idx];
 
    /* this better be empty.  We just added a block thinking it was */
    Assert(dlist_is_empty(blocklist));
 
    dlist_push_head(blocklist, &block->node);
 
    slab->curBlocklistIndex = blocklist_idx;
 
    return SlabAllocSetupNewChunk(context, block, chunk, size);
}

References Assert, SlabContext::blocklist, SlabContext::blockSize, SlabContext::chunksPerBlock, SlabContext::curBlocklistIndex, dclist_count(), dclist_pop_head_node(), dlist_container, dlist_is_empty(), dlist_push_head(), SlabContext::emptyblocks, fb(), SlabBlock::freehead, malloc, MemoryContextData::mem_allocated, MemoryContextAllocationFailure(), SlabBlock::nfree, SlabBlock::node, SlabBlock::nunused, SlabBlock::slab, Slab_BLOCKHDRSZ, SlabAllocSetupNewChunk(), SlabBlockGetChunk, SlabBlocklistIndex(), SlabGetNextFreeChunk(), unlikely, SlabBlock::unused, and VALGRIND_MEMPOOL_ALLOC.

Referenced by SlabAlloc().

SlabAllocInvalidSize()

pg_noinline static pg_noreturn void SlabAllocInvalidSize ( MemoryContext  context, Size  size  )

static

Definition at line 634 of file slab.c.

{
    SlabContext *slab = (SlabContext *) context;
 
    elog(ERROR, "unexpected alloc chunk size %zu (expected %u)", size,
         slab->chunkSize);
}

References SlabContext::chunkSize, elog, and ERROR.

Referenced by SlabAlloc().

SlabAllocSetupNewChunk()

static void * SlabAllocSetupNewChunk ( MemoryContext  context, SlabBlock *  block, MemoryChunk *  chunk, Size  size  )

inlinestatic

Definition at line 523 of file slab.c.

{
    SlabContext *slab = (SlabContext *) context;
 
    /*
     * Check that the chunk pointer is actually somewhere on the block and is
     * aligned as expected.
     */
    Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
    Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
    Assert(SlabChunkMod(slab, block, chunk) == 0);
 
    /* Prepare to initialize the chunk header. */
    VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
 
    MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize), MCTX_SLAB_ID);
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* slab mark to catch clobber of "unused" space */
    Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
    set_sentinel(MemoryChunkGetPointer(chunk), size);
    VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) + Slab_CHUNKHDRSZ +
                               slab->chunkSize,
                               slab->fullChunkSize -
                               (slab->chunkSize + Slab_CHUNKHDRSZ));
#endif
 
#ifdef RANDOMIZE_ALLOCATED_MEMORY
    /* fill the allocated space with junk */
    randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
#endif
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    return MemoryChunkGetPointer(chunk);
}

References Assert, SlabContext::chunkSize, SlabContext::chunksPerBlock, fb(), SlabContext::fullChunkSize, MAXALIGN, MCTX_SLAB_ID, MemoryChunkGetPointer, MemoryChunkSetHdrMask(), Slab_CHUNKHDRSZ, SlabBlockGetChunk, VALGRIND_MAKE_MEM_NOACCESS, and VALGRIND_MAKE_MEM_UNDEFINED.

Referenced by SlabAlloc(), and SlabAllocFromNewBlock().

SlabBlocklistIndex()

static int32 SlabBlocklistIndex ( SlabContext *  slab, int  nfree  )

inlinestatic

Definition at line 211 of file slab.c.

{
    int32       index;
    int32       blocklist_shift = slab->blocklist_shift;
 
    Assert(nfree >= 0 && nfree <= slab->chunksPerBlock);
 
    /*
     * Determine the blocklist index based on the number of free chunks.  We
     * must ensure that 0 free chunks is dedicated to index 0.  Everything
     * else must be >= 1 and < SLAB_BLOCKLIST_COUNT.
     *
     * To make this as efficient as possible, we exploit some two's complement
     * arithmetic where we reverse the sign before bit shifting.  This results
     * in an nfree of 0 using index 0 and anything non-zero staying non-zero.
     * This is exploiting 0 and -0 being the same in two's complement.  When
     * we're done, we just need to flip the sign back over again for a
     * positive index.
     */
    index = -((-nfree) >> blocklist_shift);
 
    if (nfree == 0)
        Assert(index == 0);
    else
        Assert(index >= 1 && index < SLAB_BLOCKLIST_COUNT);
 
    return index;
}

References Assert, SlabContext::blocklist_shift, fb(), and SLAB_BLOCKLIST_COUNT.

Referenced by SlabAlloc(), SlabAllocFromNewBlock(), and SlabFree().

SlabContextCreate()

MemoryContext SlabContextCreate	(	MemoryContext	parent,
		const char *	name,
		Size	blockSize,
		Size	chunkSize
	)

Definition at line 322 of file slab.c.

{
    int         chunksPerBlock;
    Size        fullChunkSize;
    SlabContext *slab;
    int         i;
 
    /* ensure MemoryChunk's size is properly maxaligned */
    StaticAssertDecl(Slab_CHUNKHDRSZ == MAXALIGN(Slab_CHUNKHDRSZ),
                     "sizeof(MemoryChunk) is not maxaligned");
    Assert(blockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
 
    /*
     * Ensure there's enough space to store the pointer to the next free chunk
     * in the memory of the (otherwise) unused allocation.
     */
    if (chunkSize < sizeof(MemoryChunk *))
        chunkSize = sizeof(MemoryChunk *);
 
    /* length of the maxaligned chunk including the chunk header  */
#ifdef MEMORY_CONTEXT_CHECKING
    /* ensure there's always space for the sentinel byte */
    fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize + 1);
#else
    fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize);
#endif
 
    Assert(fullChunkSize <= MEMORYCHUNK_MAX_VALUE);
 
    /* compute the number of chunks that will fit on each block */
    chunksPerBlock = (blockSize - Slab_BLOCKHDRSZ) / fullChunkSize;
 
    /* Make sure the block can store at least one chunk. */
    if (chunksPerBlock == 0)
        elog(ERROR, "block size %zu for slab is too small for %zu-byte chunks",
             blockSize, chunkSize);
 
 
 
    slab = (SlabContext *) malloc(Slab_CONTEXT_HDRSZ(chunksPerBlock));
    if (slab == NULL)
    {
        MemoryContextStats(TopMemoryContext);
        ereport(ERROR,
                (errcode(ERRCODE_OUT_OF_MEMORY),
                 errmsg("out of memory"),
                 errdetail("Failed while creating memory context \"%s\".",
                           name)));
    }
 
    /*
     * Avoid writing code that can fail between here and MemoryContextCreate;
     * we'd leak the header if we ereport in this stretch.
     */
 
    /* See comments about Valgrind interactions in aset.c */
    VALGRIND_CREATE_MEMPOOL(slab, 0, false);
    /* This vchunk covers the SlabContext only */
    VALGRIND_MEMPOOL_ALLOC(slab, slab, sizeof(SlabContext));
 
    /* Fill in SlabContext-specific header fields */
    slab->chunkSize = (uint32) chunkSize;
    slab->fullChunkSize = (uint32) fullChunkSize;
    slab->blockSize = (uint32) blockSize;
    slab->chunksPerBlock = chunksPerBlock;
    slab->curBlocklistIndex = 0;
 
    /*
     * Compute a shift that guarantees that shifting chunksPerBlock with it is
     * < SLAB_BLOCKLIST_COUNT - 1.  The reason that we subtract 1 from
     * SLAB_BLOCKLIST_COUNT in this calculation is that we reserve the 0th
     * blocklist element for blocks which have no free chunks.
     *
     * We calculate the number of bits to shift by rather than a divisor to
     * divide by as performing division each time we need to find the
     * blocklist index would be much slower.
     */
    slab->blocklist_shift = 0;
    while ((slab->chunksPerBlock >> slab->blocklist_shift) >= (SLAB_BLOCKLIST_COUNT - 1))
        slab->blocklist_shift++;
 
    /* initialize the list to store empty blocks to be reused */
    dclist_init(&slab->emptyblocks);
 
    /* initialize each blocklist slot */
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
        dlist_init(&slab->blocklist[i]);
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* set the isChunkFree pointer right after the end of the context */
    slab->isChunkFree = (bool *) ((char *) slab + sizeof(SlabContext));
#endif
 
    /* Finally, do the type-independent part of context creation */
    MemoryContextCreate((MemoryContext) slab,
                        T_SlabContext,
                        MCTX_SLAB_ID,
                        parent,
                        name);
 
    return (MemoryContext) slab;
}

References Assert, SlabContext::blocklist, SlabContext::blocklist_shift, SlabContext::blockSize, SlabContext::chunkSize, SlabContext::chunksPerBlock, SlabContext::curBlocklistIndex, dclist_init(), dlist_init(), elog, SlabContext::emptyblocks, ereport, errcode(), errdetail(), errmsg(), ERROR, fb(), SlabContext::fullChunkSize, i, malloc, MAXALIGN, MCTX_SLAB_ID, MEMORYCHUNK_MAX_BLOCKOFFSET, MEMORYCHUNK_MAX_VALUE, MemoryContextCreate(), MemoryContextStats(), name, Slab_BLOCKHDRSZ, SLAB_BLOCKLIST_COUNT, Slab_CHUNKHDRSZ, Slab_CONTEXT_HDRSZ, StaticAssertDecl, TopMemoryContext, VALGRIND_CREATE_MEMPOOL, and VALGRIND_MEMPOOL_ALLOC.

Referenced by for(), ReorderBufferAllocate(), and test_random().

SlabDelete()

void SlabDelete

(

MemoryContext

context

)

Definition at line 506 of file slab.c.

{
    /* Reset to release all the SlabBlocks */
    SlabReset(context);
 
    /* Destroy the vpool -- see notes in aset.c */
    VALGRIND_DESTROY_MEMPOOL(context);
 
    /* And free the context header */
    free(context);
}

References free, SlabReset(), and VALGRIND_DESTROY_MEMPOOL.

SlabFindNextBlockListIndex()

static int32 SlabFindNextBlockListIndex ( SlabContext *  slab )

static

Definition at line 251 of file slab.c.

{
    /* start at 1 as blocklist[0] is for full blocks. */
    for (int i = 1; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        /* return the first found non-empty index */
        if (!dlist_is_empty(&slab->blocklist[i]))
            return i;
    }
 
    /* no blocks with free space */
    return 0;
}

References SlabContext::blocklist, dlist_is_empty(), i, and SLAB_BLOCKLIST_COUNT.

Referenced by SlabAlloc(), and SlabFree().

SlabFree()

void SlabFree

(

void *

pointer

)

Definition at line 729 of file slab.c.

{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
    SlabContext *slab;
    int         curBlocklistIdx;
    int         newBlocklistIdx;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /*
     * For speed reasons we just Assert that the referenced block is good.
     * Future field experience may show that this Assert had better become a
     * regular runtime test-and-elog check.
     */
    Assert(SlabBlockIsValid(block));
    slab = block->slab;
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* Test for someone scribbling on unused space in chunk */
    Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
    if (!sentinel_ok(pointer, slab->chunkSize))
        elog(WARNING, "detected write past chunk end in %s %p",
             slab->header.name, chunk);
#endif
 
    /* push this chunk onto the head of the block's free list */
    *(MemoryChunk **) pointer = block->freehead;
    block->freehead = chunk;
 
    block->nfree++;
 
    Assert(block->nfree > 0);
    Assert(block->nfree <= slab->chunksPerBlock);
 
#ifdef CLOBBER_FREED_MEMORY
    /* don't wipe the free list MemoryChunk pointer stored in the chunk */
    wipe_mem((char *) pointer + sizeof(MemoryChunk *),
             slab->chunkSize - sizeof(MemoryChunk *));
#endif
 
    curBlocklistIdx = SlabBlocklistIndex(slab, block->nfree - 1);
    newBlocklistIdx = SlabBlocklistIndex(slab, block->nfree);
 
    /*
     * Check if the block needs to be moved to another element on the
     * blocklist based on it now having 1 more free chunk.
     */
    if (unlikely(curBlocklistIdx != newBlocklistIdx))
    {
        /* do the move */
        dlist_delete_from(&slab->blocklist[curBlocklistIdx], &block->node);
        dlist_push_head(&slab->blocklist[newBlocklistIdx], &block->node);
 
        /*
         * The blocklist[curBlocklistIdx] may now be empty or we may now be
         * able to use a lower-element blocklist.  We'll need to redetermine
         * what the slab->curBlocklistIndex is if the current blocklist was
         * changed or if a lower element one was changed.  We must ensure we
         * use the list with the fullest block(s).
         */
        if (slab->curBlocklistIndex >= curBlocklistIdx)
        {
            slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
 
            /*
             * We know there must be a block with at least 1 unused chunk as
             * we just pfree'd one.  Ensure curBlocklistIndex reflects this.
             */
            Assert(slab->curBlocklistIndex > 0);
        }
    }
 
    /* Handle when a block becomes completely empty */
    if (unlikely(block->nfree == slab->chunksPerBlock))
    {
        /* remove the block */
        dlist_delete_from(&slab->blocklist[newBlocklistIdx], &block->node);
 
        /*
         * To avoid thrashing malloc/free, we keep a list of empty blocks that
         * we can reuse again instead of having to malloc a new one.
         */
        if (dclist_count(&slab->emptyblocks) < SLAB_MAXIMUM_EMPTY_BLOCKS)
            dclist_push_head(&slab->emptyblocks, &block->node);
        else
        {
            /*
             * When we have enough empty blocks stored already, we actually
             * free the block.
             */
#ifdef CLOBBER_FREED_MEMORY
            wipe_mem(block, slab->blockSize);
#endif
 
            /* As in aset.c, free block-header vchunks explicitly */
            VALGRIND_MEMPOOL_FREE(slab, block);
 
            free(block);
            slab->header.mem_allocated -= slab->blockSize;
        }
 
        /*
         * Check if we need to reset the blocklist index.  This is required
         * when the blocklist this block is on has become completely empty.
         */
        if (slab->curBlocklistIndex == newBlocklistIdx &&
            dlist_is_empty(&slab->blocklist[newBlocklistIdx]))
            slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
    }
}

References Assert, SlabContext::blocklist, SlabContext::blockSize, SlabContext::chunkSize, SlabContext::chunksPerBlock, SlabContext::curBlocklistIndex, dclist_count(), dclist_push_head(), dlist_delete_from(), dlist_is_empty(), dlist_push_head(), elog, SlabContext::emptyblocks, fb(), free, SlabBlock::freehead, SlabContext::fullChunkSize, SlabContext::header, MemoryContextData::mem_allocated, MemoryChunkGetBlock(), MemoryContextData::name, SlabBlock::nfree, SlabBlock::node, PointerGetMemoryChunk, SlabBlock::slab, Slab_CHUNKHDRSZ, SLAB_MAXIMUM_EMPTY_BLOCKS, SlabBlockIsValid, SlabBlocklistIndex(), SlabFindNextBlockListIndex(), unlikely, VALGRIND_MAKE_MEM_DEFINED, VALGRIND_MEMPOOL_FREE, and WARNING.

SlabGetChunkContext()

MemoryContext SlabGetChunkContext

(

void *

pointer

)

Definition at line 895 of file slab.c.

{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    Assert(SlabBlockIsValid(block));
 
    return &block->slab->header;
}

References Assert, fb(), SlabContext::header, MemoryChunkGetBlock(), PointerGetMemoryChunk, SlabBlock::slab, Slab_CHUNKHDRSZ, SlabBlockIsValid, VALGRIND_MAKE_MEM_DEFINED, and VALGRIND_MAKE_MEM_NOACCESS.

SlabGetChunkSpace()

Size SlabGetChunkSpace

(

void *

pointer

)

Definition at line 919 of file slab.c.

{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
    SlabContext *slab;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    Assert(SlabBlockIsValid(block));
    slab = block->slab;
 
    return slab->fullChunkSize;
}

References Assert, fb(), SlabContext::fullChunkSize, MemoryChunkGetBlock(), PointerGetMemoryChunk, SlabBlock::slab, Slab_CHUNKHDRSZ, SlabBlockIsValid, VALGRIND_MAKE_MEM_DEFINED, and VALGRIND_MAKE_MEM_NOACCESS.

SlabGetNextFreeChunk()

static MemoryChunk * SlabGetNextFreeChunk ( SlabContext *  slab, SlabBlock *  block  )

inlinestatic

Definition at line 271 of file slab.c.

{
    MemoryChunk *chunk;
 
    Assert(block->nfree > 0);
 
    if (block->freehead != NULL)
    {
        chunk = block->freehead;
 
        /*
         * Pop the chunk from the linked list of free chunks.  The pointer to
         * the next free chunk is stored in the chunk itself.
         */
        VALGRIND_MAKE_MEM_DEFINED(SlabChunkGetPointer(chunk), sizeof(MemoryChunk *));
        block->freehead = *(MemoryChunk **) SlabChunkGetPointer(chunk);
 
        /* check nothing stomped on the free chunk's memory */
        Assert(block->freehead == NULL ||
               (block->freehead >= SlabBlockGetChunk(slab, block, 0) &&
                block->freehead <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1) &&
                SlabChunkMod(slab, block, block->freehead) == 0));
    }
    else
    {
        Assert(block->nunused > 0);
 
        chunk = block->unused;
        block->unused = (MemoryChunk *) (((char *) block->unused) + slab->fullChunkSize);
        block->nunused--;
    }
 
    block->nfree--;
 
    return chunk;
}

References Assert, SlabContext::chunksPerBlock, fb(), SlabBlock::freehead, SlabContext::fullChunkSize, SlabBlock::nfree, SlabBlock::nunused, SlabBlockGetChunk, SlabChunkGetPointer, SlabBlock::unused, and VALGRIND_MAKE_MEM_DEFINED.

Referenced by SlabAlloc(), and SlabAllocFromNewBlock().

SlabIsEmpty()

bool SlabIsEmpty

(

MemoryContext

context

)

Definition at line 944 of file slab.c.

{
    Assert(SlabIsValid((SlabContext *) context));
 
    return (context->mem_allocated == 0);
}

References Assert, MemoryContextData::mem_allocated, and SlabIsValid.

SlabRealloc()

void * SlabRealloc	(	void *	pointer,
		Size	size,
		int	flags
	)

Definition at line 858 of file slab.c.

{
    MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
    SlabBlock  *block;
    SlabContext *slab;
 
    /* Allow access to the chunk header. */
    VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
 
    block = MemoryChunkGetBlock(chunk);
 
    /* Disallow access to the chunk header. */
    VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
 
    /*
     * Try to verify that we have a sane block pointer: the block header
     * should reference a slab context.  (We use a test-and-elog, not just
     * Assert, because it seems highly likely that we're here in error in the
     * first place.)
     */
    if (!SlabBlockIsValid(block))
        elog(ERROR, "could not find block containing chunk %p", chunk);
    slab = block->slab;
 
    /* can't do actual realloc with slab, but let's try to be gentle */
    if (size == slab->chunkSize)
        return pointer;
 
    elog(ERROR, "slab allocator does not support realloc()");
    return NULL;                /* keep compiler quiet */
}

References SlabContext::chunkSize, elog, ERROR, fb(), MemoryChunkGetBlock(), PointerGetMemoryChunk, SlabBlock::slab, Slab_CHUNKHDRSZ, SlabBlockIsValid, VALGRIND_MAKE_MEM_DEFINED, and VALGRIND_MAKE_MEM_NOACCESS.

SlabReset()

void SlabReset

(

MemoryContext

context

)

Definition at line 436 of file slab.c.

{
    SlabContext *slab = (SlabContext *) context;
    dlist_mutable_iter miter;
    int         i;
 
    Assert(SlabIsValid(slab));
 
#ifdef MEMORY_CONTEXT_CHECKING
    /* Check for corruption and leaks before freeing */
    SlabCheck(context);
#endif
 
    /* release any retained empty blocks */
    dclist_foreach_modify(miter, &slab->emptyblocks)
    {
        SlabBlock  *block = dlist_container(SlabBlock, node, miter.cur);
 
        dclist_delete_from(&slab->emptyblocks, miter.cur);
 
#ifdef CLOBBER_FREED_MEMORY
        wipe_mem(block, slab->blockSize);
#endif
 
        /* As in aset.c, free block-header vchunks explicitly */
        VALGRIND_MEMPOOL_FREE(slab, block);
 
        free(block);
        context->mem_allocated -= slab->blockSize;
    }
 
    /* walk over blocklist and free the blocks */
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        dlist_foreach_modify(miter, &slab->blocklist[i])
        {
            SlabBlock  *block = dlist_container(SlabBlock, node, miter.cur);
 
            dlist_delete(miter.cur);
 
#ifdef CLOBBER_FREED_MEMORY
            wipe_mem(block, slab->blockSize);
#endif
 
            /* As in aset.c, free block-header vchunks explicitly */
            VALGRIND_MEMPOOL_FREE(slab, block);
 
            free(block);
            context->mem_allocated -= slab->blockSize;
        }
    }
 
    /*
     * Instruct Valgrind to throw away all the vchunks associated with this
     * context, except for the one covering the SlabContext.  This gets rid of
     * the vchunks for whatever user data is getting discarded by the context
     * reset.
     */
    VALGRIND_MEMPOOL_TRIM(slab, slab, sizeof(SlabContext));
 
    slab->curBlocklistIndex = 0;
 
    Assert(context->mem_allocated == 0);
}

References Assert, SlabContext::blocklist, SlabContext::blockSize, SlabContext::curBlocklistIndex, dclist_delete_from(), dclist_foreach_modify, dlist_container, dlist_delete(), dlist_foreach_modify, SlabContext::emptyblocks, fb(), free, i, MemoryContextData::mem_allocated, SLAB_BLOCKLIST_COUNT, SlabIsValid, VALGRIND_MEMPOOL_FREE, and VALGRIND_MEMPOOL_TRIM.

Referenced by SlabDelete().

SlabStats()

void SlabStats	(	MemoryContext	context,
		MemoryStatsPrintFunc	printfunc,
		void *	passthru,
		MemoryContextCounters *	totals,
		bool	print_to_stderr
	)

Definition at line 961 of file slab.c.

{
    SlabContext *slab = (SlabContext *) context;
    Size        nblocks = 0;
    Size        freechunks = 0;
    Size        totalspace;
    Size        freespace = 0;
    int         i;
 
    Assert(SlabIsValid(slab));
 
    /* Include context header in totalspace */
    totalspace = Slab_CONTEXT_HDRSZ(slab->chunksPerBlock);
 
    /* Add the space consumed by blocks in the emptyblocks list */
    totalspace += dclist_count(&slab->emptyblocks) * slab->blockSize;
 
    for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
    {
        dlist_iter  iter;
 
        dlist_foreach(iter, &slab->blocklist[i])
        {
            SlabBlock  *block = dlist_container(SlabBlock, node, iter.cur);
 
            nblocks++;
            totalspace += slab->blockSize;
            freespace += slab->fullChunkSize * block->nfree;
            freechunks += block->nfree;
        }
    }
 
    if (printfunc)
    {
        char        stats_string[200];
 
        /* XXX should we include free chunks on empty blocks? */
        snprintf(stats_string, sizeof(stats_string),
                 "%zu total in %zu blocks; %u empty blocks; %zu free (%zu chunks); %zu used",
                 totalspace, nblocks, dclist_count(&slab->emptyblocks),
                 freespace, freechunks, totalspace - freespace);
        printfunc(context, passthru, stats_string, print_to_stderr);
    }
 
    if (totals)
    {
        totals->nblocks += nblocks;
        totals->freechunks += freechunks;
        totals->totalspace += totalspace;
        totals->freespace += freespace;
    }
}

References Assert, SlabContext::blocklist, SlabContext::blockSize, SlabContext::chunksPerBlock, dlist_iter::cur, dclist_count(), dlist_container, dlist_foreach, SlabContext::emptyblocks, fb(), SlabContext::fullChunkSize, i, SlabBlock::nfree, SLAB_BLOCKLIST_COUNT, Slab_CONTEXT_HDRSZ, SlabIsValid, and snprintf.