#include "postgres.h"
#include "port/atomics.h"
#include "port/pg_bitutils.h"
#include "storage/dsm.h"
#include "storage/lwlock.h"
#include "utils/dsa.h"
#include "utils/freepage.h"
#include "utils/memutils.h"
#include "utils/resowner.h"

Include dependency graph for dsa.c:

Data Structures
struct	dsa_segment_header

struct	dsa_area_span

struct	dsa_area_pool

struct	dsa_area_control

struct	dsa_segment_map

struct	dsa_area

Macros
#define	DSA_NUM_SEGMENTS_AT_EACH_SIZE 2

#define	DSA_MAX_SEGMENTS Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))

#define	DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)

#define	DSA_PAGES_PER_SUPERBLOCK 16

#define	DSA_SEGMENT_HEADER_MAGIC 0x0ce26608

#define	DSA_MAKE_POINTER(segment_number, offset) (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) \| (offset))

#define	DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)

#define	DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)

#define	DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)

#define	DSA_NUM_SEGMENT_BINS 16

#define	DSA_AREA_LOCK(area) (&area->control->lock)

#define	DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)

#define	NextFreeObjectIndex(object) (* (uint16 *) (object))

#define	DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)

#define	DSA_SCLASS_BLOCK_OF_SPANS 0

#define	DSA_SCLASS_SPAN_LARGE 1

#define	DSA_SIZE_CLASS_MAP_QUANTUM 8

#define	DSA_FULLNESS_CLASSES 4

#define	DsaAreaPoolToDsaPointer(area, p) DSA_MAKE_POINTER(0, (char ) p - (char ) area->control)

#define	DSA_SPAN_NOTHING_FREE ((uint16) -1)

#define	DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)

#define	get_segment_index(area, segment_map_ptr) (segment_map_ptr - &area->segment_maps[0])

Typedefs
typedef size_t	dsa_segment_index

Functions
static size_t	contiguous_pages_to_segment_bin (size_t n)

static void	init_span (dsa_area area, dsa_pointer span_pointer, dsa_area_pool pool, dsa_pointer start, size_t npages, uint16 size_class)

static bool	transfer_first_span (dsa_area area, dsa_area_pool pool, int fromclass, int toclass)

static dsa_pointer	alloc_object (dsa_area *area, int size_class)

static bool	ensure_active_superblock (dsa_area area, dsa_area_pool pool, int size_class)

static dsa_segment_map *	get_segment_by_index (dsa_area *area, dsa_segment_index index)

static void	destroy_superblock (dsa_area *area, dsa_pointer span_pointer)

static void	unlink_span (dsa_area area, dsa_area_span span)

static void	add_span_to_fullness_class (dsa_area area, dsa_area_span span, dsa_pointer span_pointer, int fclass)

static void	unlink_segment (dsa_area area, dsa_segment_map segment_map)

static dsa_segment_map *	get_best_segment (dsa_area *area, size_t npages)

static dsa_segment_map *	make_new_segment (dsa_area *area, size_t requested_pages)

static dsa_area *	create_internal (void place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment control_segment, size_t init_segment_size, size_t max_segment_size)

static dsa_area *	attach_internal (void place, dsm_segment segment, dsa_handle handle)

static void	check_for_freed_segments (dsa_area *area)

static void	check_for_freed_segments_locked (dsa_area *area)

static void	rebin_segment (dsa_area area, dsa_segment_map segment_map)

dsa_area *	dsa_create_ext (int tranche_id, size_t init_segment_size, size_t max_segment_size)

dsa_area *	dsa_create_in_place_ext (void place, size_t size, int tranche_id, dsm_segment segment, size_t init_segment_size, size_t max_segment_size)

dsa_handle	dsa_get_handle (dsa_area *area)

dsa_area *	dsa_attach (dsa_handle handle)

dsa_area *	dsa_attach_in_place (void place, dsm_segment segment)

void	dsa_on_dsm_detach_release_in_place (dsm_segment *segment, Datum place)

void	dsa_on_shmem_exit_release_in_place (int code, Datum place)

void	dsa_release_in_place (void *place)

void	dsa_pin_mapping (dsa_area *area)

dsa_pointer	dsa_allocate_extended (dsa_area *area, size_t size, int flags)

void	dsa_free (dsa_area *area, dsa_pointer dp)

void *	dsa_get_address (dsa_area *area, dsa_pointer dp)

void	dsa_pin (dsa_area *area)

void	dsa_unpin (dsa_area *area)

void	dsa_set_size_limit (dsa_area *area, size_t limit)

size_t	dsa_get_total_size (dsa_area *area)

void	dsa_trim (dsa_area *area)

void	dsa_dump (dsa_area *area)

size_t	dsa_minimum_size (void)

void	dsa_detach (dsa_area *area)

Variables
static const uint16	dsa_size_classes []

static const uint8	dsa_size_class_map []

Macro Definition Documentation

◆ DSA_AREA_LOCK

#define DSA_AREA_LOCK ( area ) (&area->control->lock)

Definition at line 132 of file dsa.c.

◆ DSA_EXTRACT_OFFSET

#define DSA_EXTRACT_OFFSET ( dp ) ((dp) & DSA_OFFSET_BITMASK)

Definition at line 99 of file dsa.c.

◆ DSA_EXTRACT_SEGMENT_NUMBER

#define DSA_EXTRACT_SEGMENT_NUMBER ( dp ) ((dp) >> DSA_OFFSET_WIDTH)

Definition at line 96 of file dsa.c.

◆ DSA_FULLNESS_CLASSES

#define DSA_FULLNESS_CLASSES 4

Definition at line 266 of file dsa.c.

◆ DSA_MAKE_POINTER

#define DSA_MAKE_POINTER	(	segment_number,
		offset
	)	(((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) \| (offset))

Definition at line 92 of file dsa.c.

◆ DSA_MAX_SEGMENTS

#define DSA_MAX_SEGMENTS Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))

Definition at line 75 of file dsa.c.

◆ DSA_NUM_SEGMENT_BINS

#define DSA_NUM_SEGMENT_BINS 16

Definition at line 111 of file dsa.c.

◆ DSA_NUM_SEGMENTS_AT_EACH_SIZE

#define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2

Definition at line 69 of file dsa.c.

◆ DSA_NUM_SIZE_CLASSES

#define DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)

Definition at line 236 of file dsa.c.

◆ DSA_OFFSET_BITMASK

#define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)

Definition at line 79 of file dsa.c.

◆ DSA_PAGES_PER_SUPERBLOCK

#define DSA_PAGES_PER_SUPERBLOCK 16

Definition at line 82 of file dsa.c.

◆ DSA_SCLASS_BLOCK_OF_SPANS

#define DSA_SCLASS_BLOCK_OF_SPANS 0

Definition at line 239 of file dsa.c.

◆ DSA_SCLASS_LOCK

#define DSA_SCLASS_LOCK	(	area,
		sclass
	)	(&area->control->pools[sclass].lock)

Definition at line 133 of file dsa.c.

◆ DSA_SCLASS_SPAN_LARGE

#define DSA_SCLASS_SPAN_LARGE 1

Definition at line 240 of file dsa.c.

◆ DSA_SEGMENT_HEADER_MAGIC

#define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608

Definition at line 89 of file dsa.c.

◆ DSA_SEGMENT_INDEX_NONE

#define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)

Definition at line 105 of file dsa.c.

◆ DSA_SIZE_CLASS_MAP_QUANTUM

#define DSA_SIZE_CLASS_MAP_QUANTUM 8

Definition at line 258 of file dsa.c.

◆ DSA_SPAN_NOTHING_FREE

#define DSA_SPAN_NOTHING_FREE ((uint16) -1)

Definition at line 375 of file dsa.c.

◆ DSA_SUPERBLOCK_SIZE

#define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)

Definition at line 376 of file dsa.c.

◆ DsaAreaPoolToDsaPointer

#define DsaAreaPoolToDsaPointer	(	area,
		p
	)	DSA_MAKE_POINTER(0, (char ) p - (char ) area->control)

Definition at line 322 of file dsa.c.

◆ get_segment_index

#define get_segment_index	(	area,
		segment_map_ptr
	)	(segment_map_ptr - &area->segment_maps[0])

Definition at line 379 of file dsa.c.

◆ NextFreeObjectIndex

#define NextFreeObjectIndex ( object ) (* (uint16 *) (object))

Definition at line 202 of file dsa.c.

Typedef Documentation

◆ dsa_segment_index

typedef size_t dsa_segment_index

Definition at line 102 of file dsa.c.

Function Documentation

◆ add_span_to_fullness_class()

static void add_span_to_fullness_class	(	dsa_area *	area,
		dsa_area_span *	span,
		dsa_pointer	span_pointer,
		int	fclass
	)

static

Definition at line 1929 of file dsa.c.

{
    dsa_area_pool *pool = dsa_get_address(area, span->pool);
 
    if (DsaPointerIsValid(pool->spans[fclass]))
    {
        dsa_area_span *head = dsa_get_address(area,
                                              pool->spans[fclass]);
 
        head->prevspan = span_pointer;
    }
    span->prevspan = InvalidDsaPointer;
    span->nextspan = pool->spans[fclass];
    pool->spans[fclass] = span_pointer;
    span->fclass = fclass;
}

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, InvalidDsaPointer, dsa_area_span::nextspan, dsa_area_span::pool, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by dsa_free().

◆ alloc_object()

static dsa_pointer alloc_object	(	dsa_area *	area,
		int	size_class
	)

inlinestatic

Definition at line 1472 of file dsa.c.

{
    dsa_area_pool *pool = &area->control->pools[size_class];
    dsa_area_span *span;
    dsa_pointer block;
    dsa_pointer result;
    char       *object;
    size_t      size;
 
    /*
     * Even though ensure_active_superblock can in turn call alloc_object if
     * it needs to allocate a new span, that's always from a different pool,
     * and the order of lock acquisition is always the same, so it's OK that
     * we hold this lock for the duration of this function.
     */
    Assert(!LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
    LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
 
    /*
     * If there's no active superblock, we must successfully obtain one or
     * fail the request.
     */
    if (!DsaPointerIsValid(pool->spans[1]) &&
        !ensure_active_superblock(area, pool, size_class))
    {
        result = InvalidDsaPointer;
    }
    else
    {
        /*
         * There should be a block in fullness class 1 at this point, and it
         * should never be completely full.  Thus we can either pop an object
         * from the free list or, failing that, initialize a new object.
         */
        Assert(DsaPointerIsValid(pool->spans[1]));
        span = (dsa_area_span *)
            dsa_get_address(area, pool->spans[1]);
        Assert(span->nallocatable > 0);
        block = span->start;
        Assert(size_class < DSA_NUM_SIZE_CLASSES);
        size = dsa_size_classes[size_class];
        if (span->firstfree != DSA_SPAN_NOTHING_FREE)
        {
            result = block + span->firstfree * size;
            object = dsa_get_address(area, result);
            span->firstfree = NextFreeObjectIndex(object);
        }
        else
        {
            result = block + span->ninitialized * size;
            ++span->ninitialized;
        }
        --span->nallocatable;
 
        /* If it's now full, move it to the highest-numbered fullness class. */
        if (span->nallocatable == 0)
            transfer_first_span(area, pool, 1, DSA_FULLNESS_CLASSES - 1);
    }
 
    Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
    LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
 
    return result;
}

References Assert(), dsa_area::control, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DsaPointerIsValid, ensure_active_superblock(), dsa_area_span::firstfree, InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::ninitialized, dsa_area_control::pools, dsa_area_pool::spans, dsa_area_span::start, and transfer_first_span().

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ attach_internal()

static dsa_area * attach_internal	(	void *	place,
		dsm_segment *	segment,
		dsa_handle	handle
	)

static

Definition at line 1326 of file dsa.c.

{
    dsa_area_control *control;
    dsa_area   *area;
    dsa_segment_map *segment_map;
 
    control = (dsa_area_control *) place;
    Assert(control->handle == handle);
    Assert(control->segment_handles[0] == handle);
    Assert(control->segment_header.magic ==
           (DSA_SEGMENT_HEADER_MAGIC ^ handle ^ 0));
 
    /* Build the backend-local area object. */
    area = palloc(sizeof(dsa_area));
    area->control = control;
    area->resowner = CurrentResourceOwner;
    memset(&area->segment_maps[0], 0,
           sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
    area->high_segment_index = 0;
 
    /* Set up the segment map for this process's mapping. */
    segment_map = &area->segment_maps[0];
    segment_map->segment = segment; /* NULL for in-place */
    segment_map->mapped_address = place;
    segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
    segment_map->fpm = (FreePageManager *)
        (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)));
    segment_map->pagemap = (dsa_pointer *)
        (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)) +
         MAXALIGN(sizeof(FreePageManager)));
 
    /* Bump the reference count. */
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    if (control->refcnt == 0)
    {
        /* We can't attach to a DSA area that has already been destroyed. */
        ereport(ERROR,
                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                 errmsg("could not attach to dynamic shared area")));
    }
    ++control->refcnt;
    area->freed_segment_counter = area->control->freed_segment_counter;
    LWLockRelease(DSA_AREA_LOCK(area));
 
    return area;
}

References Assert(), dsa_area::control, CurrentResourceOwner, DSA_AREA_LOCK, DSA_MAX_SEGMENTS, DSA_SEGMENT_HEADER_MAGIC, ereport, errcode(), errmsg(), ERROR, dsa_segment_map::fpm, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_segment_map::mapped_address, MAXALIGN, dsa_segment_map::pagemap, palloc(), dsa_area_control::refcnt, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area_control::segment_header, and dsa_area::segment_maps.

Referenced by dsa_attach(), and dsa_attach_in_place().

◆ check_for_freed_segments()

static void check_for_freed_segments ( dsa_area * area )

static

Definition at line 2252 of file dsa.c.

{
    size_t      freed_segment_counter;
 
    /*
     * Any other process that has freed a segment has incremented
     * freed_segment_counter while holding an LWLock, and that must precede
     * any backend creating a new segment in the same slot while holding an
     * LWLock, and that must precede the creation of any dsa_pointer pointing
     * into the new segment which might reach us here, and the caller must
     * have sent the dsa_pointer to this process using appropriate memory
     * synchronization (some kind of locking or atomic primitive or system
     * call).  So all we need to do on the reading side is ask for the load of
     * freed_segment_counter to follow the caller's load of the dsa_pointer it
     * has, and we can be sure to detect any segments that had been freed as
     * of the time that the dsa_pointer reached this process.
     */
    pg_read_barrier();
    freed_segment_counter = area->control->freed_segment_counter;
    if (unlikely(area->freed_segment_counter != freed_segment_counter))
    {
        /* Check all currently mapped segments to find what's been freed. */
        LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
        check_for_freed_segments_locked(area);
        LWLockRelease(DSA_AREA_LOCK(area));
    }
}

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), pg_read_barrier, and unlikely.

Referenced by dsa_free(), and dsa_get_address().

◆ check_for_freed_segments_locked()

static void check_for_freed_segments_locked ( dsa_area * area )

static

Definition at line 2288 of file dsa.c.

{
    size_t      freed_segment_counter;
    int         i;
 
    Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
    freed_segment_counter = area->control->freed_segment_counter;
    if (unlikely(area->freed_segment_counter != freed_segment_counter))
    {
        for (i = 0; i <= area->high_segment_index; ++i)
        {
            if (area->segment_maps[i].header != NULL &&
                area->segment_maps[i].header->freed)
            {
                dsm_detach(area->segment_maps[i].segment);
                area->segment_maps[i].segment = NULL;
                area->segment_maps[i].header = NULL;
                area->segment_maps[i].mapped_address = NULL;
            }
        }
        area->freed_segment_counter = freed_segment_counter;
    }
}

References Assert(), dsa_area::control, DSA_AREA_LOCK, dsm_detach(), dsa_segment_header::freed, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_segment_map::header, dsa_area::high_segment_index, i, LWLockHeldByMe(), dsa_segment_map::mapped_address, dsa_segment_map::segment, dsa_area::segment_maps, and unlikely.

Referenced by check_for_freed_segments(), destroy_superblock(), dsa_dump(), and get_best_segment().

◆ contiguous_pages_to_segment_bin()

static size_t contiguous_pages_to_segment_bin ( size_t n )

inlinestatic

Definition at line 119 of file dsa.c.

{
    size_t      bin;
 
    if (n == 0)
        bin = 0;
    else
        bin = pg_leftmost_one_pos_size_t(n) + 1;
 
    return Min(bin, DSA_NUM_SEGMENT_BINS - 1);
}

References DSA_NUM_SEGMENT_BINS, Min, and pg_leftmost_one_pos_size_t.

Referenced by create_internal(), get_best_segment(), make_new_segment(), and rebin_segment().

◆ create_internal()

static dsa_area * create_internal	(	void *	place,
		size_t	size,
		int	tranche_id,
		dsm_handle	control_handle,
		dsm_segment *	control_segment,
		size_t	init_segment_size,
		size_t	max_segment_size
	)

static

Definition at line 1218 of file dsa.c.

{
    dsa_area_control *control;
    dsa_area   *area;
    dsa_segment_map *segment_map;
    size_t      usable_pages;
    size_t      total_pages;
    size_t      metadata_bytes;
    int         i;
 
    /* Check the initial and maximum block sizes */
    Assert(init_segment_size >= DSA_MIN_SEGMENT_SIZE);
    Assert(max_segment_size >= init_segment_size);
    Assert(max_segment_size <= DSA_MAX_SEGMENT_SIZE);
 
    /* Sanity check on the space we have to work in. */
    if (size < dsa_minimum_size())
        elog(ERROR, "dsa_area space must be at least %zu, but %zu provided",
             dsa_minimum_size(), size);
 
    /* Now figure out how much space is usable */
    total_pages = size / FPM_PAGE_SIZE;
    metadata_bytes =
        MAXALIGN(sizeof(dsa_area_control)) +
        MAXALIGN(sizeof(FreePageManager)) +
        total_pages * sizeof(dsa_pointer);
    /* Add padding up to next page boundary. */
    if (metadata_bytes % FPM_PAGE_SIZE != 0)
        metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
    Assert(metadata_bytes <= size);
    usable_pages = (size - metadata_bytes) / FPM_PAGE_SIZE;
 
    /*
     * Initialize the dsa_area_control object located at the start of the
     * space.
     */
    control = (dsa_area_control *) place;
    memset(place, 0, sizeof(*control));
    control->segment_header.magic =
        DSA_SEGMENT_HEADER_MAGIC ^ control_handle ^ 0;
    control->segment_header.next = DSA_SEGMENT_INDEX_NONE;
    control->segment_header.prev = DSA_SEGMENT_INDEX_NONE;
    control->segment_header.usable_pages = usable_pages;
    control->segment_header.freed = false;
    control->segment_header.size = size;
    control->handle = control_handle;
    control->init_segment_size = init_segment_size;
    control->max_segment_size = max_segment_size;
    control->max_total_segment_size = (size_t) -1;
    control->total_segment_size = size;
    control->segment_handles[0] = control_handle;
    for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
        control->segment_bins[i] = DSA_SEGMENT_INDEX_NONE;
    control->refcnt = 1;
    control->lwlock_tranche_id = tranche_id;
 
    /*
     * Create the dsa_area object that this backend will use to access the
     * area.  Other backends will need to obtain their own dsa_area object by
     * attaching.
     */
    area = palloc(sizeof(dsa_area));
    area->control = control;
    area->resowner = CurrentResourceOwner;
    memset(area->segment_maps, 0, sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
    area->high_segment_index = 0;
    area->freed_segment_counter = 0;
    LWLockInitialize(&control->lock, control->lwlock_tranche_id);
    for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
        LWLockInitialize(DSA_SCLASS_LOCK(area, i),
                         control->lwlock_tranche_id);
 
    /* Set up the segment map for this process's mapping. */
    segment_map = &area->segment_maps[0];
    segment_map->segment = control_segment;
    segment_map->mapped_address = place;
    segment_map->header = (dsa_segment_header *) place;
    segment_map->fpm = (FreePageManager *)
        (segment_map->mapped_address +
         MAXALIGN(sizeof(dsa_area_control)));
    segment_map->pagemap = (dsa_pointer *)
        (segment_map->mapped_address +
         MAXALIGN(sizeof(dsa_area_control)) +
         MAXALIGN(sizeof(FreePageManager)));
 
    /* Set up the free page map. */
    FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
    /* There can be 0 usable pages if size is dsa_minimum_size(). */
 
    if (usable_pages > 0)
        FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
                           usable_pages);
 
    /* Put this segment into the appropriate bin. */
    control->segment_bins[contiguous_pages_to_segment_bin(usable_pages)] = 0;
    segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
 
    return area;
}

References Assert(), dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, CurrentResourceOwner, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_MIN_SEGMENT_SIZE, dsa_minimum_size(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, elog, ERROR, dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area::freed_segment_counter, FreePageManagerInitialize(), FreePageManagerPut(), dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, i, dsa_area_control::init_segment_size, dsa_area_control::lock, dsa_area_control::lwlock_tranche_id, LWLockInitialize(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area_control::max_segment_size, dsa_area_control::max_total_segment_size, MAXALIGN, dsa_segment_header::next, dsa_segment_map::pagemap, palloc(), dsa_segment_header::prev, dsa_area_control::refcnt, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area_control::segment_header, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

Referenced by dsa_create_ext(), and dsa_create_in_place_ext().

◆ destroy_superblock()

static void destroy_superblock	(	dsa_area *	area,
		dsa_pointer	span_pointer
	)

static

Definition at line 1837 of file dsa.c.

{
    dsa_area_span *span = dsa_get_address(area, span_pointer);
    int         size_class = span->size_class;
    dsa_segment_map *segment_map;
 
 
    /* Remove it from its fullness class list. */
    unlink_span(area, span);
 
    /*
     * Note: Here we acquire the area lock while we already hold a per-pool
     * lock.  We never hold the area lock and then take a pool lock, or we
     * could deadlock.
     */
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    check_for_freed_segments_locked(area);
    segment_map =
        get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(span->start));
    FreePageManagerPut(segment_map->fpm,
                       DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
                       span->npages);
    /* Check if the segment is now entirely free. */
    if (fpm_largest(segment_map->fpm) == segment_map->header->usable_pages)
    {
        dsa_segment_index index = get_segment_index(area, segment_map);
 
        /* If it's not the segment with extra control data, free it. */
        if (index != 0)
        {
            /*
             * Give it back to the OS, and allow other backends to detect that
             * they need to detach.
             */
            unlink_segment(area, segment_map);
            segment_map->header->freed = true;
            Assert(area->control->total_segment_size >=
                   segment_map->header->size);
            area->control->total_segment_size -=
                segment_map->header->size;
            dsm_unpin_segment(dsm_segment_handle(segment_map->segment));
            dsm_detach(segment_map->segment);
            area->control->segment_handles[index] = DSM_HANDLE_INVALID;
            ++area->control->freed_segment_counter;
            segment_map->segment = NULL;
            segment_map->header = NULL;
            segment_map->mapped_address = NULL;
        }
    }
 
    /* Move segment to appropriate bin if necessary. */
    if (segment_map->header != NULL)
        rebin_segment(area, segment_map);
 
    LWLockRelease(DSA_AREA_LOCK(area));
 
    /*
     * Span-of-spans blocks store the span which describes them within the
     * block itself, so freeing the storage implicitly frees the descriptor
     * also.  If this is a block of any other type, we need to separately free
     * the span object also.  This recursive call to dsa_free will acquire the
     * span pool's lock.  We can't deadlock because the acquisition order is
     * always some other pool and then the span pool.
     */
    if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
        dsa_free(area, span_pointer);
}

References Assert(), check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, dsm_detach(), DSM_HANDLE_INVALID, dsm_segment_handle(), dsm_unpin_segment(), dsa_segment_map::fpm, fpm_largest, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area_control::freed_segment_counter, FreePageManagerPut(), get_segment_by_index(), get_segment_index, dsa_segment_map::header, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_span::npages, rebin_segment(), dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_segment_header::size, dsa_area_span::size_class, dsa_area_span::start, dsa_area_control::total_segment_size, unlink_segment(), unlink_span(), and dsa_segment_header::usable_pages.

Referenced by dsa_free(), and dsa_trim().

◆ dsa_allocate_extended()

dsa_pointer dsa_allocate_extended	(	dsa_area *	area,
		size_t	size,
		int	flags
	)

Definition at line 671 of file dsa.c.

{
    uint16      size_class;
    dsa_pointer start_pointer;
    dsa_segment_map *segment_map;
    dsa_pointer result;
 
    Assert(size > 0);
 
    /* Sanity check on huge individual allocation size. */
    if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
        ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
        elog(ERROR, "invalid DSA memory alloc request size %zu", size);
 
    /*
     * If bigger than the largest size class, just grab a run of pages from
     * the free page manager, instead of allocating an object from a pool.
     * There will still be a span, but it's a special class of span that
     * manages this whole allocation and simply gives all pages back to the
     * free page manager when dsa_free is called.
     */
    if (size > dsa_size_classes[lengthof(dsa_size_classes) - 1])
    {
        size_t      npages = fpm_size_to_pages(size);
        size_t      first_page;
        dsa_pointer span_pointer;
        dsa_area_pool *pool = &area->control->pools[DSA_SCLASS_SPAN_LARGE];
 
        /* Obtain a span object. */
        span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
        if (!DsaPointerIsValid(span_pointer))
        {
            /* Raise error unless asked not to. */
            if ((flags & DSA_ALLOC_NO_OOM) == 0)
                ereport(ERROR,
                        (errcode(ERRCODE_OUT_OF_MEMORY),
                         errmsg("out of memory"),
                         errdetail("Failed on DSA request of size %zu.",
                                   size)));
            return InvalidDsaPointer;
        }
 
        LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
 
        /* Find a segment from which to allocate. */
        segment_map = get_best_segment(area, npages);
        if (segment_map == NULL)
            segment_map = make_new_segment(area, npages);
        if (segment_map == NULL)
        {
            /* Can't make any more segments: game over. */
            LWLockRelease(DSA_AREA_LOCK(area));
            dsa_free(area, span_pointer);
 
            /* Raise error unless asked not to. */
            if ((flags & DSA_ALLOC_NO_OOM) == 0)
                ereport(ERROR,
                        (errcode(ERRCODE_OUT_OF_MEMORY),
                         errmsg("out of memory"),
                         errdetail("Failed on DSA request of size %zu.",
                                   size)));
            return InvalidDsaPointer;
        }
 
        /*
         * Ask the free page manager for a run of pages.  This should always
         * succeed, since both get_best_segment and make_new_segment should
         * only return a non-NULL pointer if it actually contains enough
         * contiguous freespace.  If it does fail, something in our backend
         * private state is out of whack, so use FATAL to kill the process.
         */
        if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
            elog(FATAL,
                 "dsa_allocate could not find %zu free pages", npages);
        LWLockRelease(DSA_AREA_LOCK(area));
 
        start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
                                         first_page * FPM_PAGE_SIZE);
 
        /* Initialize span and pagemap. */
        LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
                      LW_EXCLUSIVE);
        init_span(area, span_pointer, pool, start_pointer, npages,
                  DSA_SCLASS_SPAN_LARGE);
        segment_map->pagemap[first_page] = span_pointer;
        LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
 
        /* Zero-initialize the memory if requested. */
        if ((flags & DSA_ALLOC_ZERO) != 0)
            memset(dsa_get_address(area, start_pointer), 0, size);
 
        return start_pointer;
    }
 
    /* Map allocation to a size class. */
    if (size < lengthof(dsa_size_class_map) * DSA_SIZE_CLASS_MAP_QUANTUM)
    {
        int         mapidx;
 
        /* For smaller sizes we have a lookup table... */
        mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
                  DSA_SIZE_CLASS_MAP_QUANTUM) - 1;
        size_class = dsa_size_class_map[mapidx];
    }
    else
    {
        uint16      min;
        uint16      max;
 
        /* ... and for the rest we search by binary chop. */
        min = dsa_size_class_map[lengthof(dsa_size_class_map) - 1];
        max = lengthof(dsa_size_classes) - 1;
 
        while (min < max)
        {
            uint16      mid = (min + max) / 2;
            uint16      class_size = dsa_size_classes[mid];
 
            if (class_size < size)
                min = mid + 1;
            else
                max = mid;
        }
 
        size_class = min;
    }
    Assert(size <= dsa_size_classes[size_class]);
    Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
 
    /* Attempt to allocate an object from the appropriate pool. */
    result = alloc_object(area, size_class);
 
    /* Check for failure to allocate. */
    if (!DsaPointerIsValid(result))
    {
        /* Raise error unless asked not to. */
        if ((flags & DSA_ALLOC_NO_OOM) == 0)
            ereport(ERROR,
                    (errcode(ERRCODE_OUT_OF_MEMORY),
                     errmsg("out of memory"),
                     errdetail("Failed on DSA request of size %zu.", size)));
        return InvalidDsaPointer;
    }
 
    /* Zero-initialize the memory if requested. */
    if ((flags & DSA_ALLOC_ZERO) != 0)
        memset(dsa_get_address(area, result), 0, size);
 
    return result;
}

References alloc_object(), AllocHugeSizeIsValid, AllocSizeIsValid, Assert(), dsa_area::control, DSA_ALLOC_HUGE, DSA_ALLOC_NO_OOM, DSA_ALLOC_ZERO, DSA_AREA_LOCK, dsa_free(), dsa_get_address(), DSA_MAKE_POINTER, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_class_map, DSA_SIZE_CLASS_MAP_QUANTUM, dsa_size_classes, DsaPointerIsValid, elog, ereport, errcode(), errdetail(), errmsg(), ERROR, FATAL, dsa_segment_map::fpm, FPM_PAGE_SIZE, fpm_size_to_pages, FreePageManagerGet(), get_best_segment(), get_segment_index, init_span(), InvalidDsaPointer, lengthof, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), make_new_segment(), dsa_segment_map::pagemap, and dsa_area_control::pools.

Referenced by dshash_create(), pagetable_allocate(), and resize().

◆ dsa_attach()

dsa_area * dsa_attach ( dsa_handle handle )

Definition at line 510 of file dsa.c.

{
    dsm_segment *segment;
    dsa_area   *area;
 
    /*
     * An area handle is really a DSM segment handle for the first segment, so
     * we go ahead and attach to that.
     */
    segment = dsm_attach(handle);
    if (segment == NULL)
        ereport(ERROR,
                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                 errmsg("could not attach to dynamic shared area")));
 
    area = attach_internal(dsm_segment_address(segment), segment, handle);
 
    /* Clean up when the control segment detaches. */
    on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                  PointerGetDatum(dsm_segment_address(segment)));
 
    return area;
}

References attach_internal(), dsa_on_dsm_detach_release_in_place(), dsm_attach(), dsm_segment_address(), ereport, errcode(), errmsg(), ERROR, on_dsm_detach(), and PointerGetDatum().

Referenced by AtProcExit_memstats_cleanup(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), pg_get_process_memory_contexts(), ProcessGetMemoryContextInterrupt(), and TidStoreAttach().

◆ dsa_attach_in_place()

dsa_area * dsa_attach_in_place	(	void *	place,
		dsm_segment *	segment
	)

Definition at line 545 of file dsa.c.

{
    dsa_area   *area;
 
    area = attach_internal(place, NULL, DSA_HANDLE_INVALID);
 
    /*
     * Clean up when the control segment detaches, if a containing DSM segment
     * was provided.
     */
    if (segment != NULL)
        on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                      PointerGetDatum(place));
 
    return area;
}

References attach_internal(), DSA_HANDLE_INVALID, dsa_on_dsm_detach_release_in_place(), on_dsm_detach(), and PointerGetDatum().

Referenced by AttachSession(), ParallelQueryMain(), and pgstat_attach_shmem().

◆ dsa_create_ext()

dsa_area * dsa_create_ext	(	int	tranche_id,
		size_t	init_segment_size,
		size_t	max_segment_size
	)

Definition at line 421 of file dsa.c.

{
    dsm_segment *segment;
    dsa_area   *area;
 
    /*
     * Create the DSM segment that will hold the shared control object and the
     * first segment of usable space.
     */
    segment = dsm_create(init_segment_size, 0);
 
    /*
     * All segments backing this area are pinned, so that DSA can explicitly
     * control their lifetime (otherwise a newly created segment belonging to
     * this area might be freed when the only backend that happens to have it
     * mapped in ends, corrupting the area).
     */
    dsm_pin_segment(segment);
 
    /* Create a new DSA area with the control object in this segment. */
    area = create_internal(dsm_segment_address(segment),
                           init_segment_size,
                           tranche_id,
                           dsm_segment_handle(segment), segment,
                           init_segment_size, max_segment_size);
 
    /* Clean up when the control segment detaches. */
    on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                  PointerGetDatum(dsm_segment_address(segment)));
 
    return area;
}

References create_internal(), dsa_on_dsm_detach_release_in_place(), dsm_create(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), on_dsm_detach(), and PointerGetDatum().

Referenced by TidStoreCreateShared().

◆ dsa_create_in_place_ext()

dsa_area * dsa_create_in_place_ext	(	void *	place,
		size_t	size,
		int	tranche_id,
		dsm_segment *	segment,
		size_t	init_segment_size,
		size_t	max_segment_size
	)

Definition at line 471 of file dsa.c.

{
    dsa_area   *area;
 
    area = create_internal(place, size, tranche_id,
                           DSM_HANDLE_INVALID, NULL,
                           init_segment_size, max_segment_size);
 
    /*
     * Clean up when the control segment detaches, if a containing DSM segment
     * was provided.
     */
    if (segment != NULL)
        on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                      PointerGetDatum(place));
 
    return area;
}

References create_internal(), dsa_on_dsm_detach_release_in_place(), DSM_HANDLE_INVALID, on_dsm_detach(), and PointerGetDatum().

◆ dsa_detach()

void dsa_detach ( dsa_area * area )

Definition at line 1952 of file dsa.c.

{
    int         i;
 
    /* Detach from all segments. */
    for (i = 0; i <= area->high_segment_index; ++i)
        if (area->segment_maps[i].segment != NULL)
            dsm_detach(area->segment_maps[i].segment);
 
    /*
     * Note that 'detaching' (= detaching from DSM segments) doesn't include
     * 'releasing' (= adjusting the reference count).  It would be nice to
     * combine these operations, but client code might never get around to
     * calling dsa_detach because of an error path, and a detach hook on any
     * particular segment is too late to detach other segments in the area
     * without risking a 'leak' warning in the non-error path.
     */
 
    /* Free the backend-local area object. */
    pfree(area);
}

References dsm_detach(), dsa_area::high_segment_index, i, pfree(), dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AtProcExit_memstats_cleanup(), DetachSession(), ExecParallelCleanup(), ParallelQueryMain(), pgstat_detach_shmem(), StatsShmemInit(), test_basic(), test_dsa_basic(), test_dsa_resowners(), test_empty(), test_random(), TidStoreDestroy(), and TidStoreDetach().

◆ dsa_dump()

void dsa_dump ( dsa_area * area )

Definition at line 1088 of file dsa.c.

{
    size_t      i,
                j;
 
    /*
     * Note: This gives an inconsistent snapshot as it acquires and releases
     * individual locks as it goes...
     */
 
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    check_for_freed_segments_locked(area);
    fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
    fprintf(stderr, "  max_total_segment_size: %zu\n",
            area->control->max_total_segment_size);
    fprintf(stderr, "  total_segment_size: %zu\n",
            area->control->total_segment_size);
    fprintf(stderr, "  refcnt: %d\n", area->control->refcnt);
    fprintf(stderr, "  pinned: %c\n", area->control->pinned ? 't' : 'f');
    fprintf(stderr, "  segment bins:\n");
    for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
    {
        if (area->control->segment_bins[i] != DSA_SEGMENT_INDEX_NONE)
        {
            dsa_segment_index segment_index;
 
            if (i == 0)
                fprintf(stderr,
                        "    segment bin %zu (no contiguous free pages):\n", i);
            else
                fprintf(stderr,
                        "    segment bin %zu (at least %d contiguous pages free):\n",
                        i, 1 << (i - 1));
            segment_index = area->control->segment_bins[i];
            while (segment_index != DSA_SEGMENT_INDEX_NONE)
            {
                dsa_segment_map *segment_map;
 
                segment_map =
                    get_segment_by_index(area, segment_index);
 
                fprintf(stderr,
                        "      segment index %zu, usable_pages = %zu, "
                        "contiguous_pages = %zu, mapped at %p\n",
                        segment_index,
                        segment_map->header->usable_pages,
                        fpm_largest(segment_map->fpm),
                        segment_map->mapped_address);
                segment_index = segment_map->header->next;
            }
        }
    }
    LWLockRelease(DSA_AREA_LOCK(area));
 
    fprintf(stderr, "  pools:\n");
    for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
    {
        bool        found = false;
 
        LWLockAcquire(DSA_SCLASS_LOCK(area, i), LW_EXCLUSIVE);
        for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
            if (DsaPointerIsValid(area->control->pools[i].spans[j]))
                found = true;
        if (found)
        {
            if (i == DSA_SCLASS_BLOCK_OF_SPANS)
                fprintf(stderr, "    pool for blocks of span objects:\n");
            else if (i == DSA_SCLASS_SPAN_LARGE)
                fprintf(stderr, "    pool for large object spans:\n");
            else
                fprintf(stderr,
                        "    pool for size class %zu (object size %hu bytes):\n",
                        i, dsa_size_classes[i]);
            for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
            {
                if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
                    fprintf(stderr, "      fullness class %zu is empty\n", j);
                else
                {
                    dsa_pointer span_pointer = area->control->pools[i].spans[j];
 
                    fprintf(stderr, "      fullness class %zu:\n", j);
                    while (DsaPointerIsValid(span_pointer))
                    {
                        dsa_area_span *span;
 
                        span = dsa_get_address(area, span_pointer);
                        fprintf(stderr,
                                "        span descriptor at "
                                DSA_POINTER_FORMAT ", superblock at "
                                DSA_POINTER_FORMAT
                                ", pages = %zu, objects free = %hu/%hu\n",
                                span_pointer, span->start, span->npages,
                                span->nallocatable, span->nmax);
                        span_pointer = span->nextspan;
                    }
                }
            }
        }
        LWLockRelease(DSA_SCLASS_LOCK(area, i));
    }
}

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_POINTER_FORMAT, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DSA_SEGMENT_INDEX_NONE, dsa_size_classes, DsaPointerIsValid, dsa_segment_map::fpm, fpm_largest, fprintf, get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, i, j, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_control::max_total_segment_size, dsa_area_span::nallocatable, dsa_segment_header::next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_control::pinned, dsa_area_control::pools, dsa_area_control::refcnt, dsa_area_control::segment_bins, dsa_area_pool::spans, dsa_area_span::start, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

◆ dsa_free()

void dsa_free	(	dsa_area *	area,
		dsa_pointer	dp
	)

Definition at line 826 of file dsa.c.

{
    dsa_segment_map *segment_map;
    int         pageno;
    dsa_pointer span_pointer;
    dsa_area_span *span;
    char       *superblock;
    char       *object;
    size_t      size;
    int         size_class;
 
    /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
    check_for_freed_segments(area);
 
    /* Locate the object, span and pool. */
    segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
    pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
    span_pointer = segment_map->pagemap[pageno];
    span = dsa_get_address(area, span_pointer);
    superblock = dsa_get_address(area, span->start);
    object = dsa_get_address(area, dp);
    size_class = span->size_class;
    size = dsa_size_classes[size_class];
 
    /*
     * Special case for large objects that live in a special span: we return
     * those pages directly to the free page manager and free the span.
     */
    if (span->size_class == DSA_SCLASS_SPAN_LARGE)
    {
 
#ifdef CLOBBER_FREED_MEMORY
        memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
#endif
 
        /* Give pages back to free page manager. */
        LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
        FreePageManagerPut(segment_map->fpm,
                           DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
                           span->npages);
 
        /* Move segment to appropriate bin if necessary. */
        rebin_segment(area, segment_map);
        LWLockRelease(DSA_AREA_LOCK(area));
 
        /* Unlink span. */
        LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
                      LW_EXCLUSIVE);
        unlink_span(area, span);
        LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
        /* Free the span object so it can be reused. */
        dsa_free(area, span_pointer);
        return;
    }
 
#ifdef CLOBBER_FREED_MEMORY
    memset(object, 0x7f, size);
#endif
 
    LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
 
    /* Put the object on the span's freelist. */
    Assert(object >= superblock);
    Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
    Assert((object - superblock) % size == 0);
    NextFreeObjectIndex(object) = span->firstfree;
    span->firstfree = (object - superblock) / size;
    ++span->nallocatable;
 
    /*
     * See if the span needs to moved to a different fullness class, or be
     * freed so its pages can be given back to the segment.
     */
    if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
    {
        /*
         * The block was completely full and is located in the
         * highest-numbered fullness class, which is never scanned for free
         * chunks.  We must move it to the next-lower fullness class.
         */
        unlink_span(area, span);
        add_span_to_fullness_class(area, span, span_pointer,
                                   DSA_FULLNESS_CLASSES - 2);
 
        /*
         * If this is the only span, and there is no active span, then we
         * should probably move this span to fullness class 1.  (Otherwise if
         * you allocate exactly all the objects in the only span, it moves to
         * class 3, then you free them all, it moves to 2, and then is given
         * back, leaving no active span).
         */
    }
    else if (span->nallocatable == span->nmax &&
             (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
    {
        /*
         * This entire block is free, and it's not the active block for this
         * size class.  Return the memory to the free page manager. We don't
         * do this for the active block to prevent hysteresis: if we
         * repeatedly allocate and free the only chunk in the active block, it
         * will be very inefficient if we deallocate and reallocate the block
         * every time.
         */
        destroy_superblock(area, span_pointer);
    }
 
    LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
}

References add_span_to_fullness_class(), Assert(), check_for_freed_segments(), destroy_superblock(), DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SUPERBLOCK_SIZE, dsa_area_span::fclass, dsa_area_span::firstfree, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerPut(), get_segment_by_index(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::nmax, dsa_area_span::npages, dsa_segment_map::pagemap, dsa_area_span::prevspan, rebin_segment(), dsa_area_span::size_class, dsa_area_span::start, and unlink_span().

Referenced by delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_free(), dshash_create(), dshash_destroy(), ExecHashTableDetach(), ExecHashTableDetachBatch(), ExecParallelCleanup(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashRepartitionFirst(), ExecParallelReinitialize(), find_or_make_matching_shared_tupledesc(), free_memorycontextstate_dsa(), pagetable_free(), pgstat_free_entry(), resize(), RT_FREE(), RT_FREE_LEAF(), RT_FREE_NODE(), tbm_free_shared_area(), test_dsa_basic(), and test_dsa_resowners().

◆ dsa_get_address()

void * dsa_get_address	(	dsa_area *	area,
		dsa_pointer	dp
	)

Definition at line 942 of file dsa.c.

{
    dsa_segment_index index;
    size_t      offset;
 
    /* Convert InvalidDsaPointer to NULL. */
    if (!DsaPointerIsValid(dp))
        return NULL;
 
    /* Process any requests to detach from freed segments. */
    check_for_freed_segments(area);
 
    /* Break the dsa_pointer into its components. */
    index = DSA_EXTRACT_SEGMENT_NUMBER(dp);
    offset = DSA_EXTRACT_OFFSET(dp);
    Assert(index < DSA_MAX_SEGMENTS);
 
    /* Check if we need to cause this segment to be mapped in. */
    if (unlikely(area->segment_maps[index].mapped_address == NULL))
    {
        /* Call for effect (we don't need the result). */
        get_segment_by_index(area, index);
    }
 
    return area->segment_maps[index].mapped_address + offset;
}

References Assert(), check_for_freed_segments(), DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_MAX_SEGMENTS, DsaPointerIsValid, get_segment_by_index(), dsa_segment_map::mapped_address, dsa_area::segment_maps, and unlikely.

◆ dsa_get_handle()

dsa_handle dsa_get_handle ( dsa_area * area )

Definition at line 498 of file dsa.c.

{
    Assert(area->control->handle != DSA_HANDLE_INVALID);
    return area->control->handle;
}

References Assert(), dsa_area::control, DSA_HANDLE_INVALID, and dsa_area_control::handle.

Referenced by init_dsm_registry(), logicalrep_launcher_attach_dshmem(), parallel_vacuum_init(), parallel_vacuum_reset_dead_items(), and ProcessGetMemoryContextInterrupt().

◆ dsa_get_total_size()

size_t dsa_get_total_size ( dsa_area * area )

Definition at line 1027 of file dsa.c.

{
    size_t      size;
 
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    size = area->control->total_segment_size;
    LWLockRelease(DSA_AREA_LOCK(area));
 
    return size;
}

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), and dsa_area_control::total_segment_size.

Referenced by RT_MEMORY_USAGE().

◆ dsa_minimum_size()

size_t dsa_minimum_size ( void )

Definition at line 1196 of file dsa.c.

{
    size_t      size;
    int         pages = 0;
 
    size = MAXALIGN(sizeof(dsa_area_control)) +
        MAXALIGN(sizeof(FreePageManager));
 
    /* Figure out how many pages we need, including the page map... */
    while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
    {
        ++pages;
        size += sizeof(dsa_pointer);
    }
 
    return pages * FPM_PAGE_SIZE;
}

References FPM_PAGE_SIZE, and MAXALIGN.

Referenced by create_internal(), ExecInitParallelPlan(), and pgstat_dsa_init_size().

◆ dsa_on_dsm_detach_release_in_place()

void dsa_on_dsm_detach_release_in_place	(	dsm_segment *	segment,
		Datum	place
	)

Definition at line 576 of file dsa.c.

{
    dsa_release_in_place(DatumGetPointer(place));
}

References DatumGetPointer(), and dsa_release_in_place().

Referenced by dsa_attach(), dsa_attach_in_place(), dsa_create_ext(), and dsa_create_in_place_ext().

◆ dsa_on_shmem_exit_release_in_place()

void dsa_on_shmem_exit_release_in_place	(	int	code,
		Datum	place
	)

Definition at line 590 of file dsa.c.

{
    dsa_release_in_place(DatumGetPointer(place));
}

References DatumGetPointer(), and dsa_release_in_place().

◆ dsa_pin()

void dsa_pin ( dsa_area * area )

Definition at line 975 of file dsa.c.

{
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    if (area->control->pinned)
    {
        LWLockRelease(DSA_AREA_LOCK(area));
        elog(ERROR, "dsa_area already pinned");
    }
    area->control->pinned = true;
    ++area->control->refcnt;
    LWLockRelease(DSA_AREA_LOCK(area));
}

References dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

Referenced by init_dsm_registry(), logicalrep_launcher_attach_dshmem(), ProcessGetMemoryContextInterrupt(), and StatsShmemInit().

◆ dsa_pin_mapping()

void dsa_pin_mapping ( dsa_area * area )

Definition at line 635 of file dsa.c.

{
    int         i;
 
    if (area->resowner != NULL)
    {
        area->resowner = NULL;
 
        for (i = 0; i <= area->high_segment_index; ++i)
            if (area->segment_maps[i].segment != NULL)
                dsm_pin_mapping(area->segment_maps[i].segment);
    }
}

References dsm_pin_mapping(), dsa_area::high_segment_index, i, dsa_area::resowner, dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AttachSession(), GetSessionDsmHandle(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), pg_get_process_memory_contexts(), pgstat_attach_shmem(), ProcessGetMemoryContextInterrupt(), and test_create().

◆ dsa_release_in_place()

void dsa_release_in_place ( void * place )

Definition at line 605 of file dsa.c.

{
    dsa_area_control *control = (dsa_area_control *) place;
    int         i;
 
    LWLockAcquire(&control->lock, LW_EXCLUSIVE);
    Assert(control->segment_header.magic ==
           (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
    Assert(control->refcnt > 0);
    if (--control->refcnt == 0)
    {
        for (i = 0; i <= control->high_segment_index; ++i)
        {
            dsm_handle  handle;
 
            handle = control->segment_handles[i];
            if (handle != DSM_HANDLE_INVALID)
                dsm_unpin_segment(handle);
        }
    }
    LWLockRelease(&control->lock);
}

References Assert(), DSA_SEGMENT_HEADER_MAGIC, DSM_HANDLE_INVALID, dsm_unpin_segment(), dsa_area_control::handle, dsa_area_control::high_segment_index, i, dsa_area_control::lock, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_area_control::refcnt, dsa_area_control::segment_handles, and dsa_area_control::segment_header.

Referenced by dsa_on_dsm_detach_release_in_place(), dsa_on_shmem_exit_release_in_place(), and pgstat_detach_shmem().

◆ dsa_set_size_limit()

void dsa_set_size_limit	(	dsa_area *	area,
		size_t	limit
	)

Definition at line 1018 of file dsa.c.

{
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    area->control->max_total_segment_size = limit;
    LWLockRelease(DSA_AREA_LOCK(area));
}

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), and dsa_area_control::max_total_segment_size.

Referenced by StatsShmemInit().

◆ dsa_trim()

void dsa_trim ( dsa_area * area )

Definition at line 1043 of file dsa.c.

{
    int         size_class;
 
    /*
     * Trim in reverse pool order so we get to the spans-of-spans last, just
     * in case any become entirely free while processing all the other pools.
     */
    for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
    {
        dsa_area_pool *pool = &area->control->pools[size_class];
        dsa_pointer span_pointer;
 
        if (size_class == DSA_SCLASS_SPAN_LARGE)
        {
            /* Large object frees give back segments aggressively already. */
            continue;
        }
 
        /*
         * Search fullness class 1 only.  That is where we expect to find an
         * entirely empty superblock (entirely empty superblocks in other
         * fullness classes are returned to the free page map by dsa_free).
         */
        LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
        span_pointer = pool->spans[1];
        while (DsaPointerIsValid(span_pointer))
        {
            dsa_area_span *span = dsa_get_address(area, span_pointer);
            dsa_pointer next = span->nextspan;
 
            if (span->nallocatable == span->nmax)
                destroy_superblock(area, span_pointer);
 
            span_pointer = next;
        }
        LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
    }
}

References dsa_area::control, destroy_superblock(), dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DsaPointerIsValid, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_control::pools, and dsa_area_pool::spans.

◆ dsa_unpin()

void dsa_unpin ( dsa_area * area )

Definition at line 994 of file dsa.c.

{
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    Assert(area->control->refcnt > 1);
    if (!area->control->pinned)
    {
        LWLockRelease(DSA_AREA_LOCK(area));
        elog(ERROR, "dsa_area not pinned");
    }
    area->control->pinned = false;
    --area->control->refcnt;
    LWLockRelease(DSA_AREA_LOCK(area));
}

References Assert(), dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

◆ ensure_active_superblock()

static bool ensure_active_superblock	(	dsa_area *	area,
		dsa_area_pool *	pool,
		int	size_class
	)

static

Definition at line 1560 of file dsa.c.

{
    dsa_pointer span_pointer;
    dsa_pointer start_pointer;
    size_t      obsize = dsa_size_classes[size_class];
    size_t      nmax;
    int         fclass;
    size_t      npages = 1;
    size_t      first_page;
    size_t      i;
    dsa_segment_map *segment_map;
 
    Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
 
    /*
     * Compute the number of objects that will fit in a block of this size
     * class.  Span-of-spans blocks are just a single page, and the first
     * object isn't available for use because it describes the block-of-spans
     * itself.
     */
    if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
        nmax = FPM_PAGE_SIZE / obsize - 1;
    else
        nmax = DSA_SUPERBLOCK_SIZE / obsize;
 
    /*
     * If fullness class 1 is empty, try to find a span to put in it by
     * scanning higher-numbered fullness classes (excluding the last one,
     * whose blocks are certain to all be completely full).
     */
    for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
    {
        span_pointer = pool->spans[fclass];
 
        while (DsaPointerIsValid(span_pointer))
        {
            int         tfclass;
            dsa_area_span *span;
            dsa_area_span *nextspan;
            dsa_area_span *prevspan;
            dsa_pointer next_span_pointer;
 
            span = (dsa_area_span *)
                dsa_get_address(area, span_pointer);
            next_span_pointer = span->nextspan;
 
            /* Figure out what fullness class should contain this span. */
            tfclass = (nmax - span->nallocatable)
                * (DSA_FULLNESS_CLASSES - 1) / nmax;
 
            /* Look up next span. */
            if (DsaPointerIsValid(span->nextspan))
                nextspan = (dsa_area_span *)
                    dsa_get_address(area, span->nextspan);
            else
                nextspan = NULL;
 
            /*
             * If utilization has dropped enough that this now belongs in some
             * other fullness class, move it there.
             */
            if (tfclass < fclass)
            {
                /* Remove from the current fullness class list. */
                if (pool->spans[fclass] == span_pointer)
                {
                    /* It was the head; remove it. */
                    Assert(!DsaPointerIsValid(span->prevspan));
                    pool->spans[fclass] = span->nextspan;
                    if (nextspan != NULL)
                        nextspan->prevspan = InvalidDsaPointer;
                }
                else
                {
                    /* It was not the head. */
                    Assert(DsaPointerIsValid(span->prevspan));
                    prevspan = (dsa_area_span *)
                        dsa_get_address(area, span->prevspan);
                    prevspan->nextspan = span->nextspan;
                }
                if (nextspan != NULL)
                    nextspan->prevspan = span->prevspan;
 
                /* Push onto the head of the new fullness class list. */
                span->nextspan = pool->spans[tfclass];
                pool->spans[tfclass] = span_pointer;
                span->prevspan = InvalidDsaPointer;
                if (DsaPointerIsValid(span->nextspan))
                {
                    nextspan = (dsa_area_span *)
                        dsa_get_address(area, span->nextspan);
                    nextspan->prevspan = span_pointer;
                }
                span->fclass = tfclass;
            }
 
            /* Advance to next span on list. */
            span_pointer = next_span_pointer;
        }
 
        /* Stop now if we found a suitable block. */
        if (DsaPointerIsValid(pool->spans[1]))
            return true;
    }
 
    /*
     * If there are no blocks that properly belong in fullness class 1, pick
     * one from some other fullness class and move it there anyway, so that we
     * have an allocation target.  Our last choice is to transfer a block
     * that's almost empty (and might become completely empty soon if left
     * alone), but even that is better than failing, which is what we must do
     * if there are no blocks at all with freespace.
     */
    Assert(!DsaPointerIsValid(pool->spans[1]));
    for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
        if (transfer_first_span(area, pool, fclass, 1))
            return true;
    if (!DsaPointerIsValid(pool->spans[1]) &&
        transfer_first_span(area, pool, 0, 1))
        return true;
 
    /*
     * We failed to find an existing span with free objects, so we need to
     * allocate a new superblock and construct a new span to manage it.
     *
     * First, get a dsa_area_span object to describe the new superblock block
     * ... unless this allocation is for a dsa_area_span object, in which case
     * that's surely not going to work.  We handle that case by storing the
     * span describing a block-of-spans inline.
     */
    if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
    {
        span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
        if (!DsaPointerIsValid(span_pointer))
            return false;
        npages = DSA_PAGES_PER_SUPERBLOCK;
    }
 
    /* Find or create a segment and allocate the superblock. */
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    segment_map = get_best_segment(area, npages);
    if (segment_map == NULL)
    {
        segment_map = make_new_segment(area, npages);
        if (segment_map == NULL)
        {
            LWLockRelease(DSA_AREA_LOCK(area));
            return false;
        }
    }
 
    /*
     * This shouldn't happen: get_best_segment() or make_new_segment()
     * promised that we can successfully allocate npages.
     */
    if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
        elog(FATAL,
             "dsa_allocate could not find %zu free pages for superblock",
             npages);
    LWLockRelease(DSA_AREA_LOCK(area));
 
    /* Compute the start of the superblock. */
    start_pointer =
        DSA_MAKE_POINTER(get_segment_index(area, segment_map),
                         first_page * FPM_PAGE_SIZE);
 
    /*
     * If this is a block-of-spans, carve the descriptor right out of the
     * allocated space.
     */
    if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
    {
        /*
         * We have a pointer into the segment.  We need to build a dsa_pointer
         * from the segment index and offset into the segment.
         */
        span_pointer = start_pointer;
    }
 
    /* Initialize span and pagemap. */
    init_span(area, span_pointer, pool, start_pointer, npages, size_class);
    for (i = 0; i < npages; ++i)
        segment_map->pagemap[first_page + i] = span_pointer;
 
    return true;
}

References alloc_object(), Assert(), DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_MAKE_POINTER, DSA_PAGES_PER_SUPERBLOCK, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SUPERBLOCK_SIZE, DsaPointerIsValid, elog, FATAL, dsa_area_span::fclass, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerGet(), get_best_segment(), get_segment_index, i, init_span(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), make_new_segment(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_segment_map::pagemap, dsa_area_span::prevspan, dsa_area_pool::spans, and transfer_first_span().

Referenced by alloc_object().

◆ get_best_segment()

static dsa_segment_map * get_best_segment	(	dsa_area *	area,
		size_t	npages
	)

static

Definition at line 2010 of file dsa.c.

{
    size_t      bin;
 
    Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
    check_for_freed_segments_locked(area);
 
    /*
     * Start searching from the first bin that *might* have enough contiguous
     * pages.
     */
    for (bin = contiguous_pages_to_segment_bin(npages);
         bin < DSA_NUM_SEGMENT_BINS;
         ++bin)
    {
        /*
         * The minimum contiguous size that any segment in this bin should
         * have.  We'll re-bin if we see segments with fewer.
         */
        size_t      threshold = (size_t) 1 << (bin - 1);
        dsa_segment_index segment_index;
 
        /* Search this bin for a segment with enough contiguous space. */
        segment_index = area->control->segment_bins[bin];
        while (segment_index != DSA_SEGMENT_INDEX_NONE)
        {
            dsa_segment_map *segment_map;
            dsa_segment_index next_segment_index;
            size_t      contiguous_pages;
 
            segment_map = get_segment_by_index(area, segment_index);
            next_segment_index = segment_map->header->next;
            contiguous_pages = fpm_largest(segment_map->fpm);
 
            /* Not enough for the request, still enough for this bin. */
            if (contiguous_pages >= threshold && contiguous_pages < npages)
            {
                segment_index = next_segment_index;
                continue;
            }
 
            /* Re-bin it if it's no longer in the appropriate bin. */
            if (contiguous_pages < threshold)
            {
                rebin_segment(area, segment_map);
 
                /*
                 * But fall through to see if it's enough to satisfy this
                 * request anyway....
                 */
            }
 
            /* Check if we are done. */
            if (contiguous_pages >= npages)
                return segment_map;
 
            /* Continue searching the same bin. */
            segment_index = next_segment_index;
        }
    }
 
    /* Not found. */
    return NULL;
}

References Assert(), check_for_freed_segments_locked(), contiguous_pages_to_segment_bin(), dsa_area::control, DSA_AREA_LOCK, DSA_NUM_SEGMENT_BINS, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), dsa_segment_map::header, LWLockHeldByMe(), dsa_segment_header::next, rebin_segment(), and dsa_area_control::segment_bins.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ get_segment_by_index()

static dsa_segment_map * get_segment_by_index	(	dsa_area *	area,
		dsa_segment_index	index
	)

static

Definition at line 1757 of file dsa.c.

{
    if (unlikely(area->segment_maps[index].mapped_address == NULL))
    {
        dsm_handle  handle;
        dsm_segment *segment;
        dsa_segment_map *segment_map;
        ResourceOwner oldowner;
 
        /*
         * If we are reached by dsa_free or dsa_get_address, there must be at
         * least one object allocated in the referenced segment.  Otherwise,
         * their caller has a double-free or access-after-free bug, which we
         * have no hope of detecting.  So we know it's safe to access this
         * array slot without holding a lock; it won't change underneath us.
         * Furthermore, we know that we can see the latest contents of the
         * slot, as explained in check_for_freed_segments, which those
         * functions call before arriving here.
         */
        handle = area->control->segment_handles[index];
 
        /* It's an error to try to access an unused slot. */
        if (handle == DSM_HANDLE_INVALID)
            elog(ERROR,
                 "dsa_area could not attach to a segment that has been freed");
 
        oldowner = CurrentResourceOwner;
        CurrentResourceOwner = area->resowner;
        segment = dsm_attach(handle);
        CurrentResourceOwner = oldowner;
        if (segment == NULL)
            elog(ERROR, "dsa_area could not attach to segment");
        segment_map = &area->segment_maps[index];
        segment_map->segment = segment;
        segment_map->mapped_address = dsm_segment_address(segment);
        segment_map->header =
            (dsa_segment_header *) segment_map->mapped_address;
        segment_map->fpm = (FreePageManager *)
            (segment_map->mapped_address +
             MAXALIGN(sizeof(dsa_segment_header)));
        segment_map->pagemap = (dsa_pointer *)
            (segment_map->mapped_address +
             MAXALIGN(sizeof(dsa_segment_header)) +
             MAXALIGN(sizeof(FreePageManager)));
 
        /* Remember the highest index this backend has ever mapped. */
        if (area->high_segment_index < index)
            area->high_segment_index = index;
 
        Assert(segment_map->header->magic ==
               (DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ index));
    }
 
    /*
     * Callers of dsa_get_address() and dsa_free() don't hold the area lock,
     * but it's a bug in the calling code and undefined behavior if the
     * address is not live (ie if the segment might possibly have been freed,
     * they're trying to use a dangling pointer).
     *
     * For dsa.c code that holds the area lock to manipulate segment_bins
     * lists, it would be a bug if we ever reach a freed segment here.  After
     * it's marked as freed, the only thing any backend should do with it is
     * unmap it, and it should always have done that in
     * check_for_freed_segments_locked() before arriving here to resolve an
     * index to a segment_map.
     *
     * Either way we can assert that we aren't returning a freed segment.
     */
    Assert(!area->segment_maps[index].header->freed);
 
    return &area->segment_maps[index];
}

References Assert(), dsa_area::control, CurrentResourceOwner, DSA_SEGMENT_HEADER_MAGIC, dsm_attach(), DSM_HANDLE_INVALID, dsm_segment_address(), elog, ERROR, dsa_segment_map::fpm, dsa_segment_header::freed, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, dsa_segment_header::magic, dsa_segment_map::mapped_address, MAXALIGN, dsa_segment_map::pagemap, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area::segment_maps, and unlikely.

Referenced by destroy_superblock(), dsa_dump(), dsa_free(), dsa_get_address(), get_best_segment(), make_new_segment(), rebin_segment(), and unlink_segment().

◆ init_span()

static void init_span	(	dsa_area *	area,
		dsa_pointer	span_pointer,
		dsa_area_pool *	pool,
		dsa_pointer	start,
		size_t	npages,
		uint16	size_class
	)

static

Definition at line 1377 of file dsa.c.

{
    dsa_area_span *span = dsa_get_address(area, span_pointer);
    size_t      obsize = dsa_size_classes[size_class];
 
    /*
     * The per-pool lock must be held because we manipulate the span list for
     * this pool.
     */
    Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
 
    /* Push this span onto the front of the span list for fullness class 1. */
    if (DsaPointerIsValid(pool->spans[1]))
    {
        dsa_area_span *head = (dsa_area_span *)
            dsa_get_address(area, pool->spans[1]);
 
        head->prevspan = span_pointer;
    }
    span->pool = DsaAreaPoolToDsaPointer(area, pool);
    span->nextspan = pool->spans[1];
    span->prevspan = InvalidDsaPointer;
    pool->spans[1] = span_pointer;
 
    span->start = start;
    span->npages = npages;
    span->size_class = size_class;
    span->ninitialized = 0;
    if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
    {
        /*
         * A block-of-spans contains its own descriptor, so mark one object as
         * initialized and reduce the count of allocatable objects by one.
         * Doing this here has the side effect of also reducing nmax by one,
         * which is important to make sure we free this object at the correct
         * time.
         */
        span->ninitialized = 1;
        span->nallocatable = FPM_PAGE_SIZE / obsize - 1;
    }
    else if (size_class != DSA_SCLASS_SPAN_LARGE)
        span->nallocatable = DSA_SUPERBLOCK_SIZE / obsize;
    span->firstfree = DSA_SPAN_NOTHING_FREE;
    span->nmax = span->nallocatable;
    span->fclass = 1;
}

References Assert(), dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DSA_SUPERBLOCK_SIZE, DsaAreaPoolToDsaPointer, DsaPointerIsValid, dsa_area_span::fclass, dsa_area_span::firstfree, FPM_PAGE_SIZE, InvalidDsaPointer, LWLockHeldByMe(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_area_span::ninitialized, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_span::pool, dsa_area_span::prevspan, dsa_area_span::size_class, dsa_area_pool::spans, dsa_area_span::start, and start.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ make_new_segment()

static dsa_segment_map * make_new_segment	(	dsa_area *	area,
		size_t	requested_pages
	)

static

Definition at line 2081 of file dsa.c.

{
    dsa_segment_index new_index;
    size_t      metadata_bytes;
    size_t      total_size;
    size_t      total_pages;
    size_t      usable_pages;
    dsa_segment_map *segment_map;
    dsm_segment *segment;
    ResourceOwner oldowner;
 
    Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
 
    /* Find a segment slot that is not in use (linearly for now). */
    for (new_index = 1; new_index < DSA_MAX_SEGMENTS; ++new_index)
    {
        if (area->control->segment_handles[new_index] == DSM_HANDLE_INVALID)
            break;
    }
    if (new_index == DSA_MAX_SEGMENTS)
        return NULL;
 
    /*
     * If the total size limit is already exceeded, then we exit early and
     * avoid arithmetic wraparound in the unsigned expressions below.
     */
    if (area->control->total_segment_size >=
        area->control->max_total_segment_size)
        return NULL;
 
    /*
     * The size should be at least as big as requested, and at least big
     * enough to follow a geometric series that approximately doubles the
     * total storage each time we create a new segment.  We use geometric
     * growth because the underlying DSM system isn't designed for large
     * numbers of segments (otherwise we might even consider just using one
     * DSM segment for each large allocation and for each superblock, and then
     * we wouldn't need to use FreePageManager).
     *
     * We decide on a total segment size first, so that we produce tidy
     * power-of-two sized segments.  This is a good property to have if we
     * move to huge pages in the future.  Then we work back to the number of
     * pages we can fit.
     */
    total_size = area->control->init_segment_size *
        ((size_t) 1 << (new_index / DSA_NUM_SEGMENTS_AT_EACH_SIZE));
    total_size = Min(total_size, area->control->max_segment_size);
    total_size = Min(total_size,
                     area->control->max_total_segment_size -
                     area->control->total_segment_size);
 
    total_pages = total_size / FPM_PAGE_SIZE;
    metadata_bytes =
        MAXALIGN(sizeof(dsa_segment_header)) +
        MAXALIGN(sizeof(FreePageManager)) +
        sizeof(dsa_pointer) * total_pages;
 
    /* Add padding up to next page boundary. */
    if (metadata_bytes % FPM_PAGE_SIZE != 0)
        metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
    if (total_size <= metadata_bytes)
        return NULL;
    usable_pages = (total_size - metadata_bytes) / FPM_PAGE_SIZE;
    Assert(metadata_bytes + usable_pages * FPM_PAGE_SIZE <= total_size);
 
    /* See if that is enough... */
    if (requested_pages > usable_pages)
    {
        /*
         * We'll make an odd-sized segment, working forward from the requested
         * number of pages.
         */
        usable_pages = requested_pages;
        metadata_bytes =
            MAXALIGN(sizeof(dsa_segment_header)) +
            MAXALIGN(sizeof(FreePageManager)) +
            usable_pages * sizeof(dsa_pointer);
 
        /* Add padding up to next page boundary. */
        if (metadata_bytes % FPM_PAGE_SIZE != 0)
            metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
        total_size = metadata_bytes + usable_pages * FPM_PAGE_SIZE;
 
        /* Is that too large for dsa_pointer's addressing scheme? */
        if (total_size > DSA_MAX_SEGMENT_SIZE)
            return NULL;
 
        /* Would that exceed the limit? */
        if (total_size > area->control->max_total_segment_size -
            area->control->total_segment_size)
            return NULL;
    }
 
    /* Create the segment. */
    oldowner = CurrentResourceOwner;
    CurrentResourceOwner = area->resowner;
    segment = dsm_create(total_size, 0);
    CurrentResourceOwner = oldowner;
    if (segment == NULL)
        return NULL;
    dsm_pin_segment(segment);
 
    /* Store the handle in shared memory to be found by index. */
    area->control->segment_handles[new_index] =
        dsm_segment_handle(segment);
    /* Track the highest segment index in the history of the area. */
    if (area->control->high_segment_index < new_index)
        area->control->high_segment_index = new_index;
    /* Track the highest segment index this backend has ever mapped. */
    if (area->high_segment_index < new_index)
        area->high_segment_index = new_index;
    /* Track total size of all segments. */
    area->control->total_segment_size += total_size;
    Assert(area->control->total_segment_size <=
           area->control->max_total_segment_size);
 
    /* Build a segment map for this segment in this backend. */
    segment_map = &area->segment_maps[new_index];
    segment_map->segment = segment;
    segment_map->mapped_address = dsm_segment_address(segment);
    segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
    segment_map->fpm = (FreePageManager *)
        (segment_map->mapped_address +
         MAXALIGN(sizeof(dsa_segment_header)));
    segment_map->pagemap = (dsa_pointer *)
        (segment_map->mapped_address +
         MAXALIGN(sizeof(dsa_segment_header)) +
         MAXALIGN(sizeof(FreePageManager)));
 
    /* Set up the free page map. */
    FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
    FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
                       usable_pages);
 
    /* Set up the segment header and put it in the appropriate bin. */
    segment_map->header->magic =
        DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ new_index;
    segment_map->header->usable_pages = usable_pages;
    segment_map->header->size = total_size;
    segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
    segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
    segment_map->header->next =
        area->control->segment_bins[segment_map->header->bin];
    segment_map->header->freed = false;
    area->control->segment_bins[segment_map->header->bin] = new_index;
    if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
    {
        dsa_segment_map *next =
            get_segment_by_index(area, segment_map->header->next);
 
        Assert(next->header->bin == segment_map->header->bin);
        next->header->prev = new_index;
    }
 
    return segment_map;
}

References Assert(), dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, CurrentResourceOwner, DSA_AREA_LOCK, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_NUM_SEGMENTS_AT_EACH_SIZE, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, dsm_create(), DSM_HANDLE_INVALID, dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, FreePageManagerInitialize(), FreePageManagerPut(), get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, dsa_area_control::high_segment_index, dsa_area::high_segment_index, dsa_area_control::init_segment_size, LWLockHeldByMe(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area_control::max_segment_size, dsa_area_control::max_total_segment_size, MAXALIGN, Min, next, dsa_segment_header::next, dsa_segment_map::pagemap, dsa_segment_header::prev, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, total_size, and dsa_segment_header::usable_pages.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ rebin_segment()

static void rebin_segment	(	dsa_area *	area,
		dsa_segment_map *	segment_map
	)

static

Definition at line 2316 of file dsa.c.

{
    size_t      new_bin;
    dsa_segment_index segment_index;
 
    new_bin = contiguous_pages_to_segment_bin(fpm_largest(segment_map->fpm));
    if (segment_map->header->bin == new_bin)
        return;
 
    /* Remove it from its current bin. */
    unlink_segment(area, segment_map);
 
    /* Push it onto the front of its new bin. */
    segment_index = get_segment_index(area, segment_map);
    segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
    segment_map->header->next = area->control->segment_bins[new_bin];
    segment_map->header->bin = new_bin;
    area->control->segment_bins[new_bin] = segment_index;
    if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
    {
        dsa_segment_map *next;
 
        next = get_segment_by_index(area, segment_map->header->next);
        Assert(next->header->bin == new_bin);
        next->header->prev = segment_index;
    }
}

References Assert(), dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), get_segment_index, dsa_segment_map::header, next, dsa_segment_header::next, dsa_segment_header::prev, dsa_area_control::segment_bins, and unlink_segment().

Referenced by destroy_superblock(), dsa_free(), and get_best_segment().

◆ transfer_first_span()

static bool transfer_first_span	(	dsa_area *	area,
		dsa_area_pool *	pool,
		int	fromclass,
		int	toclass
	)

static

Definition at line 1432 of file dsa.c.

{
    dsa_pointer span_pointer;
    dsa_area_span *span;
    dsa_area_span *nextspan;
 
    /* Can't do it if source list is empty. */
    span_pointer = pool->spans[fromclass];
    if (!DsaPointerIsValid(span_pointer))
        return false;
 
    /* Remove span from head of source list. */
    span = dsa_get_address(area, span_pointer);
    pool->spans[fromclass] = span->nextspan;
    if (DsaPointerIsValid(span->nextspan))
    {
        nextspan = (dsa_area_span *)
            dsa_get_address(area, span->nextspan);
        nextspan->prevspan = InvalidDsaPointer;
    }
 
    /* Add span to head of target list. */
    span->nextspan = pool->spans[toclass];
    pool->spans[toclass] = span_pointer;
    if (DsaPointerIsValid(span->nextspan))
    {
        nextspan = (dsa_area_span *)
            dsa_get_address(area, span->nextspan);
        nextspan->prevspan = span_pointer;
    }
    span->fclass = toclass;
 
    return true;
}

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, InvalidDsaPointer, dsa_area_span::nextspan, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by alloc_object(), and ensure_active_superblock().

◆ unlink_segment()

static void unlink_segment	(	dsa_area *	area,
		dsa_segment_map *	segment_map
	)

static

Definition at line 1978 of file dsa.c.

{
    if (segment_map->header->prev != DSA_SEGMENT_INDEX_NONE)
    {
        dsa_segment_map *prev;
 
        prev = get_segment_by_index(area, segment_map->header->prev);
        prev->header->next = segment_map->header->next;
    }
    else
    {
        Assert(area->control->segment_bins[segment_map->header->bin] ==
               get_segment_index(area, segment_map));
        area->control->segment_bins[segment_map->header->bin] =
            segment_map->header->next;
    }
    if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
    {
        dsa_segment_map *next;
 
        next = get_segment_by_index(area, segment_map->header->next);
        next->header->prev = segment_map->header->prev;
    }
}

References Assert(), dsa_segment_header::bin, dsa_area::control, DSA_SEGMENT_INDEX_NONE, get_segment_by_index(), get_segment_index, dsa_segment_map::header, next, dsa_segment_header::next, dsa_segment_header::prev, and dsa_area_control::segment_bins.

Referenced by destroy_superblock(), and rebin_segment().

◆ unlink_span()

static void unlink_span	(	dsa_area *	area,
		dsa_area_span *	span
	)

static

Definition at line 1906 of file dsa.c.

{
    if (DsaPointerIsValid(span->nextspan))
    {
        dsa_area_span *next = dsa_get_address(area, span->nextspan);
 
        next->prevspan = span->prevspan;
    }
    if (DsaPointerIsValid(span->prevspan))
    {
        dsa_area_span *prev = dsa_get_address(area, span->prevspan);
 
        prev->nextspan = span->nextspan;
    }
    else
    {
        dsa_area_pool *pool = dsa_get_address(area, span->pool);
 
        pool->spans[span->fclass] = span->nextspan;
    }
}

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, next, dsa_area_span::nextspan, dsa_area_span::pool, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by destroy_superblock(), and dsa_free().

Variable Documentation

◆ dsa_size_class_map

const uint8 dsa_size_class_map[]

static

Initial value:

= {
    2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
    14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
    18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
    20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
    22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
    23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
    24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
    25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
}

Definition at line 248 of file dsa.c.

Referenced by dsa_allocate_extended().

◆ dsa_size_classes

const uint16 dsa_size_classes[]

static

Initial value:

= {
    sizeof(dsa_area_span), 0,   
    8, 16, 24, 32, 40, 48, 56, 64,  
    80, 96, 112, 128,           
    160, 192, 224, 256,         
    320, 384, 448, 512,         
    640, 768, 896, 1024,        
    1280, 1560, 1816, 2048,     
    2616, 3120, 3640, 4096,     
    5456, 6552, 7280, 8192      
}

Definition at line 225 of file dsa.c.

Referenced by alloc_object(), dsa_allocate_extended(), dsa_dump(), dsa_free(), ensure_active_superblock(), and init_span().

Data Structures

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ DSA_AREA_LOCK

◆ DSA_EXTRACT_OFFSET

◆ DSA_EXTRACT_SEGMENT_NUMBER

◆ DSA_FULLNESS_CLASSES

◆ DSA_MAKE_POINTER

◆ DSA_MAX_SEGMENTS

◆ DSA_NUM_SEGMENT_BINS

◆ DSA_NUM_SEGMENTS_AT_EACH_SIZE

◆ DSA_NUM_SIZE_CLASSES

◆ DSA_OFFSET_BITMASK

◆ DSA_PAGES_PER_SUPERBLOCK

◆ DSA_SCLASS_BLOCK_OF_SPANS

◆ DSA_SCLASS_LOCK

◆ DSA_SCLASS_SPAN_LARGE

◆ DSA_SEGMENT_HEADER_MAGIC

◆ DSA_SEGMENT_INDEX_NONE

◆ DSA_SIZE_CLASS_MAP_QUANTUM

◆ DSA_SPAN_NOTHING_FREE

◆ DSA_SUPERBLOCK_SIZE

◆ DsaAreaPoolToDsaPointer

◆ get_segment_index

◆ NextFreeObjectIndex

Typedef Documentation

◆ dsa_segment_index

Function Documentation

◆ add_span_to_fullness_class()

◆ alloc_object()

◆ attach_internal()

◆ check_for_freed_segments()

◆ check_for_freed_segments_locked()

◆ contiguous_pages_to_segment_bin()

◆ create_internal()

◆ destroy_superblock()

◆ dsa_allocate_extended()

◆ dsa_attach()

◆ dsa_attach_in_place()

◆ dsa_create_ext()

◆ dsa_create_in_place_ext()

◆ dsa_detach()

◆ dsa_dump()

◆ dsa_free()

◆ dsa_get_address()

◆ dsa_get_handle()

◆ dsa_get_total_size()

◆ dsa_minimum_size()

◆ dsa_on_dsm_detach_release_in_place()

◆ dsa_on_shmem_exit_release_in_place()

◆ dsa_pin()

◆ dsa_pin_mapping()

◆ dsa_release_in_place()

◆ dsa_set_size_limit()

◆ dsa_trim()

◆ dsa_unpin()

◆ ensure_active_superblock()

◆ get_best_segment()

◆ get_segment_by_index()

◆ init_span()

◆ make_new_segment()

◆ rebin_segment()

◆ transfer_first_span()

◆ unlink_segment()

◆ unlink_span()

Variable Documentation

◆ dsa_size_class_map

◆ dsa_size_classes