shmem.c File Reference

#include "postgres.h"
#include <unistd.h>
#include "access/slru.h"
#include "fmgr.h"
#include "funcapi.h"
#include "miscadmin.h"
#include "port/pg_bitutils.h"
#include "port/pg_numa.h"
#include "storage/lwlock.h"
#include "storage/pg_shmem.h"
#include "storage/shmem.h"
#include "storage/shmem_internal.h"
#include "storage/spin.h"
#include "utils/builtins.h"
#include "utils/tuplestore.h"

Include dependency graph for shmem.c:

Go to the source code of this file.

Data Structures
struct	ShmemRequest
struct	ShmemAllocatorData
struct	ShmemIndexEnt

Macros
#define	ShmemIndexLock   (&ShmemAllocator->index_lock)
#define	SHMEM_INDEX_KEYSIZE   (48)
#define	SHMEM_INDEX_ADDITIONAL_SIZE   (128)
#define	PG_GET_SHMEM_SIZES_COLS   4
#define	PG_GET_SHMEM_NUMA_SIZES_COLS   3

Typedefs
typedef struct ShmemAllocatorData	ShmemAllocatorData

Enumerations
enum	shmem_request_state {   SRS_INITIAL , SRS_REQUESTING , SRS_INITIALIZING , SRS_ATTACHING ,   SRS_AFTER_STARTUP_ATTACH_OR_INIT , SRS_DONE }

Functions
static void *	ShmemAllocRaw (Size size, Size alignment, Size *allocated_size)
static void	CallShmemCallbacksAfterStartup (const ShmemCallbacks *callbacks)
static void	InitShmemIndexEntry (ShmemRequest *request)
static bool	AttachShmemIndexEntry (ShmemRequest *request, bool missing_ok)
Datum	pg_numa_available (PG_FUNCTION_ARGS)
void	ShmemRequestStructWithOpts (const ShmemStructOpts *options)
void	ShmemRequestInternal (ShmemStructOpts *options, ShmemRequestKind kind)
size_t	ShmemGetRequestedSize (void)
void	ShmemInitRequested (void)
void	InitShmemAllocator (PGShmemHeader *seghdr)
void	ResetShmemAllocator (void)
void *	ShmemAlloc (Size size)
void *	ShmemAllocNoError (Size size)
bool	ShmemAddrIsValid (const void *addr)
void	RegisterShmemCallbacks (const ShmemCallbacks *callbacks)
void	ShmemCallRequestCallbacks (void)
void *	ShmemInitStruct (const char *name, Size size, bool *foundPtr)
Datum	pg_get_shmem_allocations (PG_FUNCTION_ARGS)
Datum	pg_get_shmem_allocations_numa (PG_FUNCTION_ARGS)
Size	pg_get_shmem_pagesize (void)

Variables
static List *	registered_shmem_callbacks
static List *	pending_shmem_requests
static enum shmem_request_state	shmem_request_state = SRS_INITIAL
static PGShmemHeader *	ShmemSegHdr
static void *	ShmemBase
static void *	ShmemEnd
static ShmemAllocatorData *	ShmemAllocator
static HTAB *	ShmemIndex
static bool	firstNumaTouch = true

Macro Definition Documentation

PG_GET_SHMEM_NUMA_SIZES_COLS

#define PG_GET_SHMEM_NUMA_SIZES_COLS   3

PG_GET_SHMEM_SIZES_COLS

#define PG_GET_SHMEM_SIZES_COLS   4

SHMEM_INDEX_ADDITIONAL_SIZE

#define SHMEM_INDEX_ADDITIONAL_SIZE   (128)

Definition at line 262 of file shmem.c.

SHMEM_INDEX_KEYSIZE

#define SHMEM_INDEX_KEYSIZE   (48)

Definition at line 255 of file shmem.c.

ShmemIndexLock

#define ShmemIndexLock   (&ShmemAllocator->index_lock)

Definition at line 236 of file shmem.c.

Typedef Documentation

ShmemAllocatorData

typedef struct ShmemAllocatorData ShmemAllocatorData

Enumeration Type Documentation

shmem_request_state

enum shmem_request_state

Enumerator
SRS_INITIAL
SRS_REQUESTING
SRS_INITIALIZING
SRS_ATTACHING
SRS_AFTER_STARTUP_ATTACH_OR_INIT
SRS_DONE

Definition at line 184 of file shmem.c.

{
    /* Initial state */
    SRS_INITIAL,
 
    /*
     * When we start calling the shmem_request callbacks, we enter the
     * SRS_REQUESTING phase.  All ShmemRequestStruct calls happen in this
     * state.
     */
    SRS_REQUESTING,
 
    /*
     * Postmaster has finished all shmem requests, and is now initializing the
     * shared memory segment.  init_fn callbacks are called in this state.
     */
    SRS_INITIALIZING,
 
    /*
     * A postmaster child process is starting up.  attach_fn callbacks are
     * called in this state.
     */
    SRS_ATTACHING,
 
    /* An after-startup allocation or attachment is in progress */
    SRS_AFTER_STARTUP_ATTACH_OR_INIT,
 
    /* Normal state after shmem initialization / attachment */
    SRS_DONE,
};

Function Documentation

AttachShmemIndexEntry()

static bool AttachShmemIndexEntry ( ShmemRequest *  request, bool  missing_ok  )

static

Definition at line 580 of file shmem.c.

{
    const char *name = request->options->name;
    ShmemIndexEnt *index_entry;
 
    /* Look it up in the shmem index */
    index_entry = (ShmemIndexEnt *)
        hash_search(ShmemIndex, name, HASH_FIND, NULL);
    if (!index_entry)
    {
        if (!missing_ok)
            ereport(ERROR,
                    (errmsg("could not find ShmemIndex entry for data structure \"%s\"",
                            request->options->name)));
        return false;
    }
 
    /* Check that the size in the index matches the request */
    if (index_entry->size != request->options->size &&
        request->options->size != SHMEM_ATTACH_UNKNOWN_SIZE)
    {
        ereport(ERROR,
                (errmsg("shared memory struct \"%s\" was created with"
                        " different size: existing %zu, requested %zd",
                        name, index_entry->size, request->options->size)));
    }
 
    /*
     * Re-establish the caller's pointer variable, or do other actions to
     * attach depending on the kind of shmem area it is.
     */
    switch (request->kind)
    {
        case SHMEM_KIND_STRUCT:
            if (request->options->ptr)
                *(request->options->ptr) = index_entry->location;
            break;
        case SHMEM_KIND_HASH:
            shmem_hash_attach(index_entry->location, request->options);
            break;
        case SHMEM_KIND_SLRU:
            shmem_slru_attach(index_entry->location, request->options);
            break;
    }
 
    return true;
}

References ereport, errmsg, ERROR, fb(), HASH_FIND, hash_search(), name, SHMEM_ATTACH_UNKNOWN_SIZE, shmem_hash_attach(), SHMEM_KIND_HASH, SHMEM_KIND_SLRU, SHMEM_KIND_STRUCT, shmem_slru_attach(), and ShmemIndex.

Referenced by CallShmemCallbacksAfterStartup(), and ShmemInitStruct().

CallShmemCallbacksAfterStartup()

static void CallShmemCallbacksAfterStartup ( const ShmemCallbacks *  callbacks )

static

Definition at line 898 of file shmem.c.

{
    bool        found_any;
    bool        notfound_any;
 
    Assert(shmem_request_state == SRS_DONE);
    shmem_request_state = SRS_REQUESTING;
 
    /*
     * Call the request callback first.  The callback makes ShmemRequest*()
     * calls for each shmem area, adding them to pending_shmem_requests.
     */
    Assert(pending_shmem_requests == NIL);
    if (callbacks->request_fn)
        callbacks->request_fn(callbacks->opaque_arg);
    shmem_request_state = SRS_AFTER_STARTUP_ATTACH_OR_INIT;
 
    if (pending_shmem_requests == NIL)
    {
        shmem_request_state = SRS_DONE;
        return;
    }
 
    /* Hold ShmemIndexLock while we allocate all the shmem entries */
    LWLockAcquire(ShmemIndexLock, LW_EXCLUSIVE);
 
    /*
     * Check if the requested shared memory areas have already been
     * initialized.  We assume all the areas requested by the request callback
     * to form a coherent unit such that they're all already initialized or
     * none.  Otherwise it would be ambiguous which callback, init or attach,
     * to callback afterwards.
     */
    found_any = notfound_any = false;
    foreach_ptr(ShmemRequest, request, pending_shmem_requests)
    {
        if (hash_search(ShmemIndex, request->options->name, HASH_FIND, NULL))
            found_any = true;
        else
            notfound_any = true;
    }
    if (found_any && notfound_any)
        elog(ERROR, "found some but not all");
 
    /*
     * Allocate or attach all the shmem areas requested by the request_fn
     * callback.
     */
    foreach_ptr(ShmemRequest, request, pending_shmem_requests)
    {
        if (found_any)
            AttachShmemIndexEntry(request, false);
        else
            InitShmemIndexEntry(request);
 
        pfree(request->options);
    }
    list_free_deep(pending_shmem_requests);
    pending_shmem_requests = NIL;
 
    /* Finish by calling the appropriate subsystem-specific callback */
    if (found_any)
    {
        if (callbacks->attach_fn)
            callbacks->attach_fn(callbacks->opaque_arg);
    }
    else
    {
        if (callbacks->init_fn)
            callbacks->init_fn(callbacks->opaque_arg);
    }
 
    LWLockRelease(ShmemIndexLock);
    shmem_request_state = SRS_DONE;
}

References Assert, ShmemCallbacks::attach_fn, AttachShmemIndexEntry(), elog, ERROR, fb(), foreach_ptr, HASH_FIND, hash_search(), ShmemCallbacks::init_fn, InitShmemIndexEntry(), list_free_deep(), LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), NIL, ShmemCallbacks::opaque_arg, pending_shmem_requests, pfree(), ShmemCallbacks::request_fn, ShmemIndex, ShmemIndexLock, SRS_AFTER_STARTUP_ATTACH_OR_INIT, SRS_DONE, and SRS_REQUESTING.

Referenced by RegisterShmemCallbacks().

InitShmemAllocator()

void InitShmemAllocator

(

PGShmemHeader *

seghdr

)

Definition at line 637 of file shmem.c.

{
    Size        offset;
    int64       hash_nelems;
    HASHCTL     info;
    int         hash_flags;
 
#ifndef EXEC_BACKEND
    Assert(!IsUnderPostmaster);
#endif
    Assert(seghdr != NULL);
 
    if (IsUnderPostmaster)
    {
        Assert(shmem_request_state == SRS_INITIAL);
    }
    else
    {
        Assert(shmem_request_state == SRS_REQUESTING);
        shmem_request_state = SRS_INITIALIZING;
    }
 
    /*
     * We assume the pointer and offset are MAXALIGN.  Not a hard requirement,
     * but it's true today and keeps the math below simpler.
     */
    Assert(seghdr == (void *) MAXALIGN(seghdr));
    Assert(seghdr->content_offset == MAXALIGN(seghdr->content_offset));
 
    /*
     * Allocations after this point should go through ShmemAlloc, which
     * expects to allocate everything on cache line boundaries.  Make sure the
     * first allocation begins on a cache line boundary.
     */
    offset = CACHELINEALIGN(seghdr->content_offset + sizeof(ShmemAllocatorData));
    if (offset > seghdr->totalsize)
        ereport(ERROR,
                (errcode(ERRCODE_OUT_OF_MEMORY),
                 errmsg("out of shared memory (%zu bytes requested)",
                        offset)));
 
    /*
     * In postmaster or stand-alone backend, initialize the shared memory
     * allocator so that we can allocate shared memory for ShmemIndex using
     * ShmemAlloc().  In a regular backend just set up the pointers required
     * by ShmemAlloc().
     */
    ShmemAllocator = (ShmemAllocatorData *) ((char *) seghdr + seghdr->content_offset);
    if (!IsUnderPostmaster)
    {
        SpinLockInit(&ShmemAllocator->shmem_lock);
        ShmemAllocator->free_offset = offset;
        LWLockInitialize(&ShmemAllocator->index_lock, LWTRANCHE_SHMEM_INDEX);
    }
 
    ShmemSegHdr = seghdr;
    ShmemBase = seghdr;
    ShmemEnd = (char *) ShmemBase + seghdr->totalsize;
 
    /*
     * Create (or attach to) the shared memory index of shmem areas.
     *
     * This is the same initialization as ShmemInitHash() does, but we cannot
     * use ShmemInitHash() here because it relies on ShmemIndex being already
     * initialized.
     */
    hash_nelems = list_length(pending_shmem_requests) + SHMEM_INDEX_ADDITIONAL_SIZE;
 
    info.keysize = SHMEM_INDEX_KEYSIZE;
    info.entrysize = sizeof(ShmemIndexEnt);
    hash_flags = HASH_ELEM | HASH_STRINGS | HASH_FIXED_SIZE;
 
    if (!IsUnderPostmaster)
    {
        ShmemAllocator->index_size = hash_estimate_size(hash_nelems, info.entrysize);
        ShmemAllocator->index = (HASHHDR *) ShmemAlloc(ShmemAllocator->index_size);
    }
    ShmemIndex = shmem_hash_create(ShmemAllocator->index,
                                   ShmemAllocator->index_size,
                                   IsUnderPostmaster,
                                   "ShmemIndex", hash_nelems,
                                   &info, hash_flags);
    Assert(ShmemIndex != NULL);
 
    /*
     * Add an entry for ShmemIndex itself into ShmemIndex, so that it's
     * visible in the pg_shmem_allocations view
     */
    if (!IsUnderPostmaster)
    {
        bool        found;
        ShmemIndexEnt *result = (ShmemIndexEnt *)
            hash_search(ShmemIndex, "ShmemIndex", HASH_ENTER, &found);
 
        Assert(!found);
        result->size = ShmemAllocator->index_size;
        result->allocated_size = ShmemAllocator->index_size;
        result->location = ShmemAllocator->index;
    }
}

References Assert, CACHELINEALIGN, HASHCTL::entrysize, ereport, errcode(), errmsg, ERROR, fb(), ShmemAllocatorData::free_offset, HASH_ELEM, HASH_ENTER, hash_estimate_size(), HASH_FIXED_SIZE, hash_search(), HASH_STRINGS, ShmemAllocatorData::index, ShmemAllocatorData::index_lock, ShmemAllocatorData::index_size, IsUnderPostmaster, HASHCTL::keysize, list_length(), LWLockInitialize(), MAXALIGN, pending_shmem_requests, result, shmem_hash_create(), SHMEM_INDEX_ADDITIONAL_SIZE, SHMEM_INDEX_KEYSIZE, ShmemAllocatorData::shmem_lock, ShmemAlloc(), ShmemAllocator, ShmemBase, ShmemEnd, ShmemIndex, ShmemSegHdr, SpinLockInit(), SRS_INITIAL, SRS_INITIALIZING, SRS_REQUESTING, and PGShmemHeader::totalsize.

Referenced by CreateSharedMemoryAndSemaphores().

InitShmemIndexEntry()

static void InitShmemIndexEntry ( ShmemRequest *  request )

static

Definition at line 511 of file shmem.c.

{
    const char *name = request->options->name;
    ShmemIndexEnt *index_entry;
    bool        found;
    size_t      allocated_size;
    void       *structPtr;
 
    /* look it up in the shmem index */
    index_entry = (ShmemIndexEnt *)
        hash_search(ShmemIndex, name, HASH_ENTER_NULL, &found);
    if (found)
        elog(ERROR, "shared memory struct \"%s\" is already initialized", name);
    if (!index_entry)
    {
        /* tried to add it to the hash table, but there was no space */
        ereport(ERROR,
                (errcode(ERRCODE_OUT_OF_MEMORY),
                 errmsg("could not create ShmemIndex entry for data structure \"%s\"",
                        name)));
    }
 
    /*
     * We inserted the entry to the shared memory index.  Allocate requested
     * amount of shared memory for it, and initialize the index entry.
     */
    structPtr = ShmemAllocRaw(request->options->size,
                              request->options->alignment,
                              &allocated_size);
    if (structPtr == NULL)
    {
        /* out of memory; remove the failed ShmemIndex entry */
        hash_search(ShmemIndex, name, HASH_REMOVE, NULL);
        ereport(ERROR,
                (errcode(ERRCODE_OUT_OF_MEMORY),
                 errmsg("not enough shared memory for data structure"
                        " \"%s\" (%zd bytes requested)",
                        name, request->options->size)));
    }
    index_entry->size = request->options->size;
    index_entry->allocated_size = allocated_size;
    index_entry->location = structPtr;
 
    /* Initialize depending on the kind of shmem area it is */
    switch (request->kind)
    {
        case SHMEM_KIND_STRUCT:
            if (request->options->ptr)
                *(request->options->ptr) = index_entry->location;
            break;
        case SHMEM_KIND_HASH:
            shmem_hash_init(structPtr, request->options);
            break;
        case SHMEM_KIND_SLRU:
            shmem_slru_init(structPtr, request->options);
            break;
    }
}

References elog, ereport, errcode(), errmsg, ERROR, fb(), HASH_ENTER_NULL, HASH_REMOVE, hash_search(), name, shmem_hash_init(), SHMEM_KIND_HASH, SHMEM_KIND_SLRU, SHMEM_KIND_STRUCT, shmem_slru_init(), ShmemAllocRaw(), and ShmemIndex.

Referenced by CallShmemCallbacksAfterStartup(), ShmemInitRequested(), and ShmemInitStruct().

pg_get_shmem_allocations()

Datum pg_get_shmem_allocations

(

PG_FUNCTION_ARGS

)

Definition at line 1045 of file shmem.c.

{
#define PG_GET_SHMEM_SIZES_COLS 4
    ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
    HASH_SEQ_STATUS hstat;
    ShmemIndexEnt *ent;
    Size        named_allocated = 0;
    Datum       values[PG_GET_SHMEM_SIZES_COLS];
    bool        nulls[PG_GET_SHMEM_SIZES_COLS];
 
    InitMaterializedSRF(fcinfo, 0);
 
    LWLockAcquire(ShmemIndexLock, LW_SHARED);
 
    hash_seq_init(&hstat, ShmemIndex);
 
    /* output all allocated entries */
    memset(nulls, 0, sizeof(nulls));
    while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL)
    {
        values[0] = CStringGetTextDatum(ent->key);
        values[1] = Int64GetDatum((char *) ent->location - (char *) ShmemSegHdr);
        values[2] = Int64GetDatum(ent->size);
        values[3] = Int64GetDatum(ent->allocated_size);
        named_allocated += ent->allocated_size;
 
        tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
                             values, nulls);
    }
 
    /* output shared memory allocated but not counted via the shmem index */
    values[0] = CStringGetTextDatum("<anonymous>");
    nulls[1] = true;
    values[2] = Int64GetDatum(ShmemAllocator->free_offset - named_allocated);
    values[3] = values[2];
    tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
 
    /* output as-of-yet unused shared memory */
    nulls[0] = true;
    values[1] = Int64GetDatum(ShmemAllocator->free_offset);
    nulls[1] = false;
    values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemAllocator->free_offset);
    values[3] = values[2];
    tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
 
    LWLockRelease(ShmemIndexLock);
 
    return (Datum) 0;
}

References CStringGetTextDatum, fb(), ShmemAllocatorData::free_offset, hash_seq_init(), hash_seq_search(), InitMaterializedSRF(), Int64GetDatum(), LW_SHARED, LWLockAcquire(), LWLockRelease(), PG_GET_SHMEM_SIZES_COLS, ShmemAllocator, ShmemIndex, ShmemIndexLock, ShmemSegHdr, PGShmemHeader::totalsize, tuplestore_putvalues(), and values.

pg_get_shmem_allocations_numa()

Datum pg_get_shmem_allocations_numa

(

PG_FUNCTION_ARGS

)

Definition at line 1102 of file shmem.c.

{
#define PG_GET_SHMEM_NUMA_SIZES_COLS 3
    ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
    HASH_SEQ_STATUS hstat;
    ShmemIndexEnt *ent;
    Datum       values[PG_GET_SHMEM_NUMA_SIZES_COLS];
    bool        nulls[PG_GET_SHMEM_NUMA_SIZES_COLS];
    Size        os_page_size;
    void      **page_ptrs;
    int        *pages_status;
    uint64      shm_total_page_count,
                shm_ent_page_count,
                max_nodes;
    Size       *nodes;
 
    if (pg_numa_init() == -1)
        elog(ERROR, "libnuma initialization failed or NUMA is not supported on this platform");
 
    InitMaterializedSRF(fcinfo, 0);
 
    max_nodes = pg_numa_get_max_node();
    nodes = palloc_array(Size, max_nodes + 2);
 
    /*
     * Shared memory allocations can vary in size and may not align with OS
     * memory page boundaries, while NUMA queries work on pages.
     *
     * To correctly map each allocation to NUMA nodes, we need to: 1.
     * Determine the OS memory page size. 2. Align each allocation's start/end
     * addresses to page boundaries. 3. Query NUMA node information for all
     * pages spanning the allocation.
     */
    os_page_size = pg_get_shmem_pagesize();
 
    /*
     * Allocate memory for page pointers and status based on total shared
     * memory size. This simplified approach allocates enough space for all
     * pages in shared memory rather than calculating the exact requirements
     * for each segment.
     *
     * Add 1, because we don't know how exactly the segments align to OS
     * pages, so the allocation might use one more memory page. In practice
     * this is not very likely, and moreover we have more entries, each of
     * them using only fraction of the total pages.
     */
    shm_total_page_count = (ShmemSegHdr->totalsize / os_page_size) + 1;
    page_ptrs = palloc0_array(void *, shm_total_page_count);
    pages_status = palloc_array(int, shm_total_page_count);
 
    if (firstNumaTouch)
        elog(DEBUG1, "NUMA: page-faulting shared memory segments for proper NUMA readouts");
 
    LWLockAcquire(ShmemIndexLock, LW_SHARED);
 
    hash_seq_init(&hstat, ShmemIndex);
 
    /* output all allocated entries */
    while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL)
    {
        int         i;
        char       *startptr,
                   *endptr;
        Size        total_len;
 
        /*
         * Calculate the range of OS pages used by this segment. The segment
         * may start / end half-way through a page, we want to count these
         * pages too. So we align the start/end pointers down/up, and then
         * calculate the number of pages from that.
         */
        startptr = (char *) TYPEALIGN_DOWN(os_page_size, ent->location);
        endptr = (char *) TYPEALIGN(os_page_size,
                                    (char *) ent->location + ent->allocated_size);
        total_len = (endptr - startptr);
 
        shm_ent_page_count = total_len / os_page_size;
 
        /*
         * If we ever get 0xff (-1) back from kernel inquiry, then we probably
         * have a bug in mapping buffers to OS pages.
         */
        memset(pages_status, 0xff, sizeof(int) * shm_ent_page_count);
 
        /*
         * Setup page_ptrs[] with pointers to all OS pages for this segment,
         * and get the NUMA status using pg_numa_query_pages.
         *
         * In order to get reliable results we also need to touch memory
         * pages, so that inquiry about NUMA memory node doesn't return -2
         * (ENOENT, which indicates unmapped/unallocated pages).
         */
        for (i = 0; i < shm_ent_page_count; i++)
        {
            page_ptrs[i] = startptr + (i * os_page_size);
 
            if (firstNumaTouch)
                pg_numa_touch_mem_if_required(page_ptrs[i]);
 
            CHECK_FOR_INTERRUPTS();
        }
 
        if (pg_numa_query_pages(0, shm_ent_page_count, page_ptrs, pages_status) == -1)
            elog(ERROR, "failed NUMA pages inquiry status: %m");
 
        /* Count number of NUMA nodes used for this shared memory entry */
        memset(nodes, 0, sizeof(Size) * (max_nodes + 2));
 
        for (i = 0; i < shm_ent_page_count; i++)
        {
            int         s = pages_status[i];
 
            /* Ensure we are adding only valid index to the array */
            if (s >= 0 && s <= max_nodes)
            {
                /* valid NUMA node */
                nodes[s]++;
                continue;
            }
            else if (s == -2)
            {
                /* -2 means ENOENT (e.g. page was moved to swap) */
                nodes[max_nodes + 1]++;
                continue;
            }
 
            elog(ERROR, "invalid NUMA node id outside of allowed range "
                 "[0, " UINT64_FORMAT "]: %d", max_nodes, s);
        }
 
        /* no NULLs for regular nodes */
        memset(nulls, 0, sizeof(nulls));
 
        /*
         * Add one entry for each NUMA node, including those without allocated
         * memory for this segment.
         */
        for (i = 0; i <= max_nodes; i++)
        {
            values[0] = CStringGetTextDatum(ent->key);
            values[1] = Int32GetDatum(i);
            values[2] = Int64GetDatum(nodes[i] * os_page_size);
 
            tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
                                 values, nulls);
        }
 
        /* The last entry is used for pages without a NUMA node. */
        nulls[1] = true;
        values[0] = CStringGetTextDatum(ent->key);
        values[2] = Int64GetDatum(nodes[max_nodes + 1] * os_page_size);
 
        tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
                             values, nulls);
    }
 
    LWLockRelease(ShmemIndexLock);
    firstNumaTouch = false;
 
    return (Datum) 0;
}

References CHECK_FOR_INTERRUPTS, CStringGetTextDatum, DEBUG1, elog, ERROR, fb(), firstNumaTouch, hash_seq_init(), hash_seq_search(), i, InitMaterializedSRF(), Int32GetDatum(), Int64GetDatum(), LW_SHARED, LWLockAcquire(), LWLockRelease(), palloc0_array, palloc_array, PG_GET_SHMEM_NUMA_SIZES_COLS, pg_get_shmem_pagesize(), pg_numa_get_max_node(), pg_numa_init(), pg_numa_query_pages(), pg_numa_touch_mem_if_required, ShmemIndex, ShmemIndexLock, ShmemSegHdr, PGShmemHeader::totalsize, tuplestore_putvalues(), TYPEALIGN, TYPEALIGN_DOWN, UINT64_FORMAT, and values.

pg_get_shmem_pagesize()

Size pg_get_shmem_pagesize

(

void

)

Definition at line 1273 of file shmem.c.

{
    Size        os_page_size;
#ifdef WIN32
    SYSTEM_INFO sysinfo;
 
    GetSystemInfo(&sysinfo);
    os_page_size = sysinfo.dwPageSize;
#else
    os_page_size = sysconf(_SC_PAGESIZE);
#endif
 
    Assert(IsUnderPostmaster);
    Assert(huge_pages_status != HUGE_PAGES_UNKNOWN);
 
    if (huge_pages_status == HUGE_PAGES_ON)
        GetHugePageSize(&os_page_size, NULL);
 
    return os_page_size;
}

References Assert, fb(), GetHugePageSize(), HUGE_PAGES_ON, huge_pages_status, HUGE_PAGES_UNKNOWN, and IsUnderPostmaster.

Referenced by pg_buffercache_os_pages_internal(), and pg_get_shmem_allocations_numa().

pg_numa_available()

Datum pg_numa_available

(

PG_FUNCTION_ARGS

)

Definition at line 1295 of file shmem.c.

{
    PG_RETURN_BOOL(pg_numa_init() != -1);
}

References pg_numa_init(), and PG_RETURN_BOOL.

RegisterShmemCallbacks()

void RegisterShmemCallbacks

(

const ShmemCallbacks *

callbacks

)

Definition at line 873 of file shmem.c.

{
    if (shmem_request_state == SRS_DONE && IsUnderPostmaster)
    {
        /*
         * After-startup initialization or attachment.  Call the appropriate
         * callbacks immediately.
         */
        if ((callbacks->flags & SHMEM_CALLBACKS_ALLOW_AFTER_STARTUP) == 0)
            elog(ERROR, "cannot request shared memory at this time");
 
        CallShmemCallbacksAfterStartup(callbacks);
    }
    else
    {
        /* Remember the callbacks for later */
        registered_shmem_callbacks = lappend(registered_shmem_callbacks,
                                             (void *) callbacks);
    }
}

References CallShmemCallbacksAfterStartup(), elog, ERROR, ShmemCallbacks::flags, IsUnderPostmaster, lappend(), registered_shmem_callbacks, SHMEM_CALLBACKS_ALLOW_AFTER_STARTUP, and SRS_DONE.

Referenced by _PG_init().

ResetShmemAllocator()

void ResetShmemAllocator

(

void

)

Definition at line 742 of file shmem.c.

{
    Assert(!IsUnderPostmaster);
    shmem_request_state = SRS_INITIAL;
 
    pending_shmem_requests = NIL;
 
    /*
     * Note that we don't clear the registered callbacks.  We will need to
     * call them again as we restart
     */
}

References Assert, IsUnderPostmaster, NIL, pending_shmem_requests, and SRS_INITIAL.

Referenced by PostmasterStateMachine().

ShmemAddrIsValid()

bool ShmemAddrIsValid

(

const void *

addr

)

Definition at line 849 of file shmem.c.

{
    return (addr >= ShmemBase) && (addr < ShmemEnd);
}

References ShmemBase, and ShmemEnd.

Referenced by ReleasePredXact().

ShmemAlloc()

void * ShmemAlloc

(

Size

size

)

Definition at line 763 of file shmem.c.

{
    void       *newSpace;
    Size        allocated_size;
 
    newSpace = ShmemAllocRaw(size, 0, &allocated_size);
    if (!newSpace)
        ereport(ERROR,
                (errcode(ERRCODE_OUT_OF_MEMORY),
                 errmsg("out of shared memory (%zu bytes requested)",
                        size)));
    return newSpace;
}

References ereport, errcode(), errmsg, ERROR, fb(), and ShmemAllocRaw().

Referenced by InitShmemAllocator().

ShmemAllocNoError()

void * ShmemAllocNoError

(

Size

size

)

Definition at line 783 of file shmem.c.

{
    Size        allocated_size;
 
    return ShmemAllocRaw(size, 0, &allocated_size);
}

References ShmemAllocRaw().

ShmemAllocRaw()

static void * ShmemAllocRaw ( Size  size, Size  alignment, Size *  allocated_size  )

static

Definition at line 797 of file shmem.c.

{
    Size        rawStart;
    Size        newStart;
    Size        newFree;
    void       *newSpace;
 
    /*
     * Ensure all space is adequately aligned.  We used to only MAXALIGN this
     * space but experience has proved that on modern systems that is not good
     * enough.  Many parts of the system are very sensitive to critical data
     * structures getting split across cache line boundaries.  To avoid that,
     * attempt to align the beginning of the allocation to a cache line
     * boundary.  The calling code will still need to be careful about how it
     * uses the allocated space - e.g. by padding each element in an array of
     * structures out to a power-of-two size - but without this, even that
     * won't be sufficient.
     */
    if (alignment < PG_CACHE_LINE_SIZE)
        alignment = PG_CACHE_LINE_SIZE;
 
    Assert(ShmemSegHdr != NULL);
 
    SpinLockAcquire(&ShmemAllocator->shmem_lock);
 
    rawStart = ShmemAllocator->free_offset;
    newStart = TYPEALIGN(alignment, rawStart);
 
    newFree = newStart + size;
    if (newFree <= ShmemSegHdr->totalsize)
    {
        newSpace = (char *) ShmemBase + newStart;
        ShmemAllocator->free_offset = newFree;
    }
    else
        newSpace = NULL;
 
    SpinLockRelease(&ShmemAllocator->shmem_lock);
 
    /* note this assert is okay with newSpace == NULL */
    Assert(newSpace == (void *) TYPEALIGN(alignment, newSpace));
 
    *allocated_size = newFree - rawStart;
    return newSpace;
}

References Assert, fb(), ShmemAllocatorData::free_offset, PG_CACHE_LINE_SIZE, ShmemAllocatorData::shmem_lock, ShmemAllocator, ShmemBase, ShmemSegHdr, SpinLockAcquire(), SpinLockRelease(), and TYPEALIGN.

Referenced by InitShmemIndexEntry(), ShmemAlloc(), and ShmemAllocNoError().

ShmemCallRequestCallbacks()

void ShmemCallRequestCallbacks

(

void

)

Definition at line 978 of file shmem.c.

{
    ListCell   *lc;
 
    Assert(shmem_request_state == SRS_INITIAL);
    shmem_request_state = SRS_REQUESTING;
 
    foreach(lc, registered_shmem_callbacks)
    {
        const ShmemCallbacks *callbacks = (const ShmemCallbacks *) lfirst(lc);
 
        if (callbacks->request_fn)
            callbacks->request_fn(callbacks->opaque_arg);
    }
}

References Assert, fb(), lfirst, ShmemCallbacks::opaque_arg, registered_shmem_callbacks, ShmemCallbacks::request_fn, SRS_INITIAL, and SRS_REQUESTING.

Referenced by BootstrapModeMain(), PostgresSingleUserMain(), PostmasterMain(), and PostmasterStateMachine().

ShmemGetRequestedSize()

size_t ShmemGetRequestedSize

(

void

)

Definition at line 391 of file shmem.c.

{
    size_t      size;
 
    /* memory needed for the ShmemIndex */
    size = hash_estimate_size(list_length(pending_shmem_requests) + SHMEM_INDEX_ADDITIONAL_SIZE,
                              sizeof(ShmemIndexEnt));
    size = CACHELINEALIGN(size);
 
    /* memory needed for all the requested areas */
    foreach_ptr(ShmemRequest, request, pending_shmem_requests)
    {
        size_t      alignment = request->options->alignment;
 
        /* pad the start address for alignment like ShmemAllocRaw() does */
        if (alignment < PG_CACHE_LINE_SIZE)
            alignment = PG_CACHE_LINE_SIZE;
        size = TYPEALIGN(alignment, size);
 
        size = add_size(size, request->options->size);
    }
 
    return size;
}

References add_size(), CACHELINEALIGN, fb(), foreach_ptr, hash_estimate_size(), list_length(), pending_shmem_requests, PG_CACHE_LINE_SIZE, SHMEM_INDEX_ADDITIONAL_SIZE, and TYPEALIGN.

Referenced by CalculateShmemSize().

ShmemInitRequested()

void ShmemInitRequested

(

void

)

Definition at line 424 of file shmem.c.

{
    /* should be called only by the postmaster or a standalone backend */
    Assert(!IsUnderPostmaster);
    Assert(shmem_request_state == SRS_INITIALIZING);
 
    /*
     * Initialize the ShmemIndex entries and perform basic initialization of
     * all the requested memory areas.  There are no concurrent processes yet,
     * so no need for locking.
     */
    foreach_ptr(ShmemRequest, request, pending_shmem_requests)
    {
        InitShmemIndexEntry(request);
        pfree(request->options);
    }
    list_free_deep(pending_shmem_requests);
    pending_shmem_requests = NIL;
 
    /*
     * Call the subsystem-specific init callbacks to finish initialization of
     * all the areas.
     */
    foreach_ptr(const ShmemCallbacks, callbacks, registered_shmem_callbacks)
    {
        if (callbacks->init_fn)
            callbacks->init_fn(callbacks->opaque_arg);
    }
 
    shmem_request_state = SRS_DONE;
}

References Assert, fb(), foreach_ptr, InitShmemIndexEntry(), IsUnderPostmaster, list_free_deep(), NIL, pending_shmem_requests, pfree(), registered_shmem_callbacks, SRS_DONE, and SRS_INITIALIZING.

Referenced by CreateSharedMemoryAndSemaphores().

ShmemInitStruct()

void * ShmemInitStruct	(	const char *	name,
		Size	size,
		bool *	foundPtr
	)

Definition at line 1010 of file shmem.c.

{
    void       *ptr = NULL;
    ShmemStructOpts options = {
        .name = name,
        .size = size,
        .ptr = &ptr,
    };
    ShmemRequest request = {&options, SHMEM_KIND_STRUCT};
 
    Assert(shmem_request_state == SRS_DONE ||
           shmem_request_state == SRS_INITIALIZING ||
           shmem_request_state == SRS_REQUESTING);
 
    LWLockAcquire(ShmemIndexLock, LW_EXCLUSIVE);
 
    /*
     * During postmaster startup, look up the existing entry if any.
     */
    *foundPtr = false;
    if (IsUnderPostmaster)
        *foundPtr = AttachShmemIndexEntry(&request, true);
 
    /* Initialize it if not found */
    if (!*foundPtr)
        InitShmemIndexEntry(&request);
 
    LWLockRelease(ShmemIndexLock);
 
    Assert(ptr != NULL);
    return ptr;
}

References Assert, AttachShmemIndexEntry(), fb(), InitShmemIndexEntry(), IsUnderPostmaster, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), name, SHMEM_KIND_STRUCT, ShmemIndexLock, SRS_DONE, SRS_INITIALIZING, and SRS_REQUESTING.

Referenced by ShmemInitHash().

ShmemRequestInternal()

void ShmemRequestInternal	(	ShmemStructOpts *	options,
		ShmemRequestKind	kind
	)

Definition at line 336 of file shmem.c.

{
    ShmemRequest *request;
 
    /* Check the options */
    if (options->name == NULL)
        elog(ERROR, "shared memory request is missing 'name' option");
 
    if (IsUnderPostmaster)
    {
        if (options->size <= 0 && options->size != SHMEM_ATTACH_UNKNOWN_SIZE)
            elog(ERROR, "invalid size %zd for shared memory request for \"%s\"",
                 options->size, options->name);
    }
    else
    {
        if (options->size == SHMEM_ATTACH_UNKNOWN_SIZE)
            elog(ERROR, "SHMEM_ATTACH_UNKNOWN_SIZE cannot be used during startup");
        if (options->size <= 0)
            elog(ERROR, "invalid size %zd for shared memory request for \"%s\"",
                 options->size, options->name);
    }
 
    if (options->alignment != 0 && pg_nextpower2_size_t(options->alignment) != options->alignment)
        elog(ERROR, "invalid alignment %zu for shared memory request for \"%s\"",
             options->alignment, options->name);
 
    /* Check that we're in the right state */
    if (shmem_request_state != SRS_REQUESTING)
        elog(ERROR, "ShmemRequestStruct can only be called from a shmem_request callback");
 
    /* Check that it's not already registered in this process */
    foreach_ptr(ShmemRequest, existing, pending_shmem_requests)
    {
        if (strcmp(existing->options->name, options->name) == 0)
            ereport(ERROR,
                    (errmsg("shared memory struct \"%s\" is already registered",
                            options->name)));
    }
 
    /* Request looks valid, remember it */
    request = palloc(sizeof(ShmemRequest));
    request->options = options;
    request->kind = kind;
    pending_shmem_requests = lappend(pending_shmem_requests, request);
}

References elog, ereport, errmsg, ERROR, fb(), foreach_ptr, IsUnderPostmaster, lappend(), palloc(), pending_shmem_requests, pg_nextpower2_size_t, SHMEM_ATTACH_UNKNOWN_SIZE, and SRS_REQUESTING.

Referenced by ShmemRequestHashWithOpts(), ShmemRequestStructWithOpts(), and SimpleLruRequestWithOpts().

ShmemRequestStructWithOpts()

void ShmemRequestStructWithOpts

(

const ShmemStructOpts *

options

)

Definition at line 315 of file shmem.c.

{
    ShmemStructOpts *options_copy;
 
    options_copy = MemoryContextAlloc(TopMemoryContext,
                                      sizeof(ShmemStructOpts));
    memcpy(options_copy, options, sizeof(ShmemStructOpts));
 
    ShmemRequestInternal(options_copy, SHMEM_KIND_STRUCT);
}

References fb(), memcpy(), MemoryContextAlloc(), SHMEM_KIND_STRUCT, ShmemRequestInternal(), and TopMemoryContext.

Variable Documentation

firstNumaTouch

bool firstNumaTouch = true

static

Definition at line 274 of file shmem.c.

Referenced by pg_get_shmem_allocations_numa().

pending_shmem_requests

List* pending_shmem_requests

static

Definition at line 169 of file shmem.c.

Referenced by CallShmemCallbacksAfterStartup(), InitShmemAllocator(), ResetShmemAllocator(), ShmemGetRequestedSize(), ShmemInitRequested(), and ShmemRequestInternal().

registered_shmem_callbacks

List* registered_shmem_callbacks

static

Definition at line 157 of file shmem.c.

Referenced by RegisterShmemCallbacks(), ShmemCallRequestCallbacks(), and ShmemInitRequested().

shmem_request_state

enum shmem_request_state shmem_request_state = SRS_INITIAL

static

Definition at line 214 of file shmem.c.

ShmemAllocator

ShmemAllocatorData* ShmemAllocator

static

Definition at line 246 of file shmem.c.

Referenced by InitShmemAllocator(), pg_get_shmem_allocations(), and ShmemAllocRaw().

ShmemBase

void* ShmemBase

static

Definition at line 243 of file shmem.c.

Referenced by InitShmemAllocator(), ShmemAddrIsValid(), and ShmemAllocRaw().

ShmemEnd

void* ShmemEnd

static

Definition at line 244 of file shmem.c.

Referenced by InitShmemAllocator(), and ShmemAddrIsValid().

ShmemIndex

HTAB* ShmemIndex

static

Definition at line 252 of file shmem.c.

Referenced by AttachShmemIndexEntry(), CallShmemCallbacksAfterStartup(), InitShmemAllocator(), InitShmemIndexEntry(), pg_get_shmem_allocations(), and pg_get_shmem_allocations_numa().

ShmemSegHdr

PGShmemHeader* ShmemSegHdr

static

Definition at line 242 of file shmem.c.

Referenced by InitShmemAllocator(), pg_get_shmem_allocations(), pg_get_shmem_allocations_numa(), and ShmemAllocRaw().