PostgreSQL Source Code  git master
dsa.c File Reference
#include "postgres.h"
#include "port/atomics.h"
#include "storage/dsm.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
#include "storage/shmem.h"
#include "utils/dsa.h"
#include "utils/freepage.h"
#include "utils/memutils.h"
Include dependency graph for dsa.c:

Go to the source code of this file.

Data Structures

struct  dsa_segment_header
 
struct  dsa_area_span
 
struct  dsa_area_pool
 
struct  dsa_area_control
 
struct  dsa_segment_map
 
struct  dsa_area
 

Macros

#define DSA_INITIAL_SEGMENT_SIZE   ((size_t) (1 * 1024 * 1024))
 
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE   2
 
#define DSA_OFFSET_WIDTH   40 /* 1024 segments of size up to 1TB */
 
#define DSA_MAX_SEGMENTS   Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
 
#define DSA_OFFSET_BITMASK   (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
 
#define DSA_MAX_SEGMENT_SIZE   ((size_t) 1 << DSA_OFFSET_WIDTH)
 
#define DSA_PAGES_PER_SUPERBLOCK   16
 
#define DSA_SEGMENT_HEADER_MAGIC   0x0ce26608
 
#define DSA_MAKE_POINTER(segment_number, offset)   (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
 
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)   ((dp) >> DSA_OFFSET_WIDTH)
 
#define DSA_EXTRACT_OFFSET(dp)   ((dp) & DSA_OFFSET_BITMASK)
 
#define DSA_SEGMENT_INDEX_NONE   (~(dsa_segment_index)0)
 
#define DSA_NUM_SEGMENT_BINS   16
 
#define contiguous_pages_to_segment_bin(n)   Min(fls(n), DSA_NUM_SEGMENT_BINS - 1)
 
#define DSA_AREA_LOCK(area)   (&area->control->lock)
 
#define DSA_SCLASS_LOCK(area, sclass)   (&area->control->pools[sclass].lock)
 
#define NextFreeObjectIndex(object)   (* (uint16 *) (object))
 
#define DSA_NUM_SIZE_CLASSES   lengthof(dsa_size_classes)
 
#define DSA_SCLASS_BLOCK_OF_SPANS   0
 
#define DSA_SCLASS_SPAN_LARGE   1
 
#define DSA_SIZE_CLASS_MAP_QUANTUM   8
 
#define DSA_FULLNESS_CLASSES   4
 
#define DsaAreaPoolToDsaPointer(area, p)   DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
 
#define DSA_SPAN_NOTHING_FREE   ((uint16) -1)
 
#define DSA_SUPERBLOCK_SIZE   (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
 
#define get_segment_index(area, segment_map_ptr)   (segment_map_ptr - &area->segment_maps[0])
 

Typedefs

typedef size_t dsa_segment_index
 

Functions

static void init_span (dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
 
static bool transfer_first_span (dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
 
static dsa_pointer alloc_object (dsa_area *area, int size_class)
 
static bool ensure_active_superblock (dsa_area *area, dsa_area_pool *pool, int size_class)
 
static dsa_segment_mapget_segment_by_index (dsa_area *area, dsa_segment_index index)
 
static void destroy_superblock (dsa_area *area, dsa_pointer span_pointer)
 
static void unlink_span (dsa_area *area, dsa_area_span *span)
 
static void add_span_to_fullness_class (dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
 
static void unlink_segment (dsa_area *area, dsa_segment_map *segment_map)
 
static dsa_segment_mapget_best_segment (dsa_area *area, size_t npages)
 
static dsa_segment_mapmake_new_segment (dsa_area *area, size_t requested_pages)
 
static dsa_areacreate_internal (void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment)
 
static dsa_areaattach_internal (void *place, dsm_segment *segment, dsa_handle handle)
 
static void check_for_freed_segments (dsa_area *area)
 
static void check_for_freed_segments_locked (dsa_area *area)
 
dsa_areadsa_create (int tranche_id)
 
dsa_areadsa_create_in_place (void *place, size_t size, int tranche_id, dsm_segment *segment)
 
dsa_handle dsa_get_handle (dsa_area *area)
 
dsa_areadsa_attach (dsa_handle handle)
 
dsa_areadsa_attach_in_place (void *place, dsm_segment *segment)
 
void dsa_on_dsm_detach_release_in_place (dsm_segment *segment, Datum place)
 
void dsa_on_shmem_exit_release_in_place (int code, Datum place)
 
void dsa_release_in_place (void *place)
 
void dsa_pin_mapping (dsa_area *area)
 
dsa_pointer dsa_allocate_extended (dsa_area *area, size_t size, int flags)
 
void dsa_free (dsa_area *area, dsa_pointer dp)
 
void * dsa_get_address (dsa_area *area, dsa_pointer dp)
 
void dsa_pin (dsa_area *area)
 
void dsa_unpin (dsa_area *area)
 
void dsa_set_size_limit (dsa_area *area, size_t limit)
 
void dsa_trim (dsa_area *area)
 
void dsa_dump (dsa_area *area)
 
size_t dsa_minimum_size (void)
 
void dsa_detach (dsa_area *area)
 

Variables

static const uint16 dsa_size_classes []
 
static const uint8 dsa_size_class_map []
 

Macro Definition Documentation

◆ contiguous_pages_to_segment_bin

#define contiguous_pages_to_segment_bin (   n)    Min(fls(n), DSA_NUM_SEGMENT_BINS - 1)

Definition at line 140 of file dsa.c.

Referenced by create_internal(), get_best_segment(), and make_new_segment().

◆ DSA_AREA_LOCK

◆ DSA_EXTRACT_OFFSET

#define DSA_EXTRACT_OFFSET (   dp)    ((dp) & DSA_OFFSET_BITMASK)

Definition at line 121 of file dsa.c.

Referenced by destroy_superblock(), dsa_free(), and dsa_get_address().

◆ DSA_EXTRACT_SEGMENT_NUMBER

#define DSA_EXTRACT_SEGMENT_NUMBER (   dp)    ((dp) >> DSA_OFFSET_WIDTH)

Definition at line 118 of file dsa.c.

Referenced by destroy_superblock(), dsa_free(), and dsa_get_address().

◆ DSA_FULLNESS_CLASSES

#define DSA_FULLNESS_CLASSES   4

Definition at line 277 of file dsa.c.

Referenced by alloc_object(), dsa_dump(), dsa_free(), and ensure_active_superblock().

◆ DSA_INITIAL_SEGMENT_SIZE

#define DSA_INITIAL_SEGMENT_SIZE   ((size_t) (1 * 1024 * 1024))

Definition at line 68 of file dsa.c.

Referenced by create_internal(), dsa_create(), and make_new_segment().

◆ DSA_MAKE_POINTER

#define DSA_MAKE_POINTER (   segment_number,
  offset 
)    (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))

Definition at line 114 of file dsa.c.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ DSA_MAX_SEGMENT_SIZE

#define DSA_MAX_SEGMENT_SIZE   ((size_t) 1 << DSA_OFFSET_WIDTH)

Definition at line 101 of file dsa.c.

Referenced by make_new_segment().

◆ DSA_MAX_SEGMENTS

#define DSA_MAX_SEGMENTS   Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))

Definition at line 94 of file dsa.c.

Referenced by attach_internal(), create_internal(), dsa_get_address(), and make_new_segment().

◆ DSA_NUM_SEGMENT_BINS

#define DSA_NUM_SEGMENT_BINS   16

Definition at line 133 of file dsa.c.

Referenced by create_internal(), dsa_dump(), and get_best_segment().

◆ DSA_NUM_SEGMENTS_AT_EACH_SIZE

#define DSA_NUM_SEGMENTS_AT_EACH_SIZE   2

Definition at line 77 of file dsa.c.

Referenced by make_new_segment().

◆ DSA_NUM_SIZE_CLASSES

#define DSA_NUM_SIZE_CLASSES   lengthof(dsa_size_classes)

Definition at line 247 of file dsa.c.

Referenced by alloc_object(), create_internal(), dsa_dump(), and dsa_trim().

◆ DSA_OFFSET_BITMASK

#define DSA_OFFSET_BITMASK   (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)

Definition at line 98 of file dsa.c.

◆ DSA_OFFSET_WIDTH

#define DSA_OFFSET_WIDTH   40 /* 1024 segments of size up to 1TB */

Definition at line 87 of file dsa.c.

◆ DSA_PAGES_PER_SUPERBLOCK

#define DSA_PAGES_PER_SUPERBLOCK   16

Definition at line 104 of file dsa.c.

Referenced by ensure_active_superblock().

◆ DSA_SCLASS_BLOCK_OF_SPANS

#define DSA_SCLASS_BLOCK_OF_SPANS   0

◆ DSA_SCLASS_LOCK

#define DSA_SCLASS_LOCK (   area,
  sclass 
)    (&area->control->pools[sclass].lock)

◆ DSA_SCLASS_SPAN_LARGE

#define DSA_SCLASS_SPAN_LARGE   1

Definition at line 251 of file dsa.c.

Referenced by dsa_allocate_extended(), dsa_dump(), dsa_free(), dsa_trim(), and init_span().

◆ DSA_SEGMENT_HEADER_MAGIC

#define DSA_SEGMENT_HEADER_MAGIC   0x0ce26608

◆ DSA_SEGMENT_INDEX_NONE

#define DSA_SEGMENT_INDEX_NONE   (~(dsa_segment_index)0)

Definition at line 127 of file dsa.c.

Referenced by create_internal(), dsa_dump(), get_best_segment(), make_new_segment(), and unlink_segment().

◆ DSA_SIZE_CLASS_MAP_QUANTUM

#define DSA_SIZE_CLASS_MAP_QUANTUM   8

Definition at line 269 of file dsa.c.

Referenced by dsa_allocate_extended().

◆ DSA_SPAN_NOTHING_FREE

#define DSA_SPAN_NOTHING_FREE   ((uint16) -1)

Definition at line 377 of file dsa.c.

Referenced by alloc_object(), and init_span().

◆ DSA_SUPERBLOCK_SIZE

#define DSA_SUPERBLOCK_SIZE   (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)

Definition at line 378 of file dsa.c.

Referenced by dsa_free(), ensure_active_superblock(), and init_span().

◆ DsaAreaPoolToDsaPointer

#define DsaAreaPoolToDsaPointer (   area,
 
)    DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)

Definition at line 329 of file dsa.c.

Referenced by init_span().

◆ get_segment_index

#define get_segment_index (   area,
  segment_map_ptr 
)    (segment_map_ptr - &area->segment_maps[0])

◆ NextFreeObjectIndex

#define NextFreeObjectIndex (   object)    (* (uint16 *) (object))

Definition at line 213 of file dsa.c.

Referenced by alloc_object(), and dsa_free().

Typedef Documentation

◆ dsa_segment_index

typedef size_t dsa_segment_index

Definition at line 124 of file dsa.c.

Function Documentation

◆ add_span_to_fullness_class()

static void add_span_to_fullness_class ( dsa_area area,
dsa_area_span span,
dsa_pointer  span_pointer,
int  fclass 
)
static

Definition at line 1887 of file dsa.c.

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, InvalidDsaPointer, dsa_area_span::nextspan, dsa_area_span::pool, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by dsa_free().

1890 {
1891  dsa_area_pool *pool = dsa_get_address(area, span->pool);
1892 
1893  if (DsaPointerIsValid(pool->spans[fclass]))
1894  {
1895  dsa_area_span *head = dsa_get_address(area,
1896  pool->spans[fclass]);
1897 
1898  head->prevspan = span_pointer;
1899  }
1900  span->prevspan = InvalidDsaPointer;
1901  span->nextspan = pool->spans[fclass];
1902  pool->spans[fclass] = span_pointer;
1903  span->fclass = fclass;
1904 }
dsa_pointer nextspan
Definition: dsa.c:198
#define InvalidDsaPointer
Definition: dsa.h:78
uint16 fclass
Definition: dsa.c:206
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
dsa_pointer prevspan
Definition: dsa.c:197
dsa_pointer pool
Definition: dsa.c:196
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81

◆ alloc_object()

static dsa_pointer alloc_object ( dsa_area area,
int  size_class 
)
inlinestatic

Definition at line 1437 of file dsa.c.

References Assert, dsa_area::control, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DsaPointerIsValid, ensure_active_superblock(), dsa_area_span::firstfree, InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::ninitialized, dsa_area_control::pools, dsa_area_pool::spans, dsa_area_span::start, and transfer_first_span().

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

1438 {
1439  dsa_area_pool *pool = &area->control->pools[size_class];
1440  dsa_area_span *span;
1441  dsa_pointer block;
1442  dsa_pointer result;
1443  char *object;
1444  size_t size;
1445 
1446  /*
1447  * Even though ensure_active_superblock can in turn call alloc_object if
1448  * it needs to allocate a new span, that's always from a different pool,
1449  * and the order of lock acquisition is always the same, so it's OK that
1450  * we hold this lock for the duration of this function.
1451  */
1452  Assert(!LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1453  LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
1454 
1455  /*
1456  * If there's no active superblock, we must successfully obtain one or
1457  * fail the request.
1458  */
1459  if (!DsaPointerIsValid(pool->spans[1]) &&
1460  !ensure_active_superblock(area, pool, size_class))
1461  {
1462  result = InvalidDsaPointer;
1463  }
1464  else
1465  {
1466  /*
1467  * There should be a block in fullness class 1 at this point, and it
1468  * should never be completely full. Thus we can either pop an object
1469  * from the free list or, failing that, initialize a new object.
1470  */
1471  Assert(DsaPointerIsValid(pool->spans[1]));
1472  span = (dsa_area_span *)
1473  dsa_get_address(area, pool->spans[1]);
1474  Assert(span->nallocatable > 0);
1475  block = span->start;
1476  Assert(size_class < DSA_NUM_SIZE_CLASSES);
1477  size = dsa_size_classes[size_class];
1478  if (span->firstfree != DSA_SPAN_NOTHING_FREE)
1479  {
1480  result = block + span->firstfree * size;
1481  object = dsa_get_address(area, result);
1482  span->firstfree = NextFreeObjectIndex(object);
1483  }
1484  else
1485  {
1486  result = block + span->ninitialized * size;
1487  ++span->ninitialized;
1488  }
1489  --span->nallocatable;
1490 
1491  /* If it's now full, move it to the highest-numbered fullness class. */
1492  if (span->nallocatable == 0)
1493  transfer_first_span(area, pool, 1, DSA_FULLNESS_CLASSES - 1);
1494  }
1495 
1496  Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1497  LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
1498 
1499  return result;
1500 }
#define DSA_SPAN_NOTHING_FREE
Definition: dsa.c:377
#define InvalidDsaPointer
Definition: dsa.h:78
uint16 ninitialized
Definition: dsa.c:202
bool LWLockHeldByMe(LWLock *l)
Definition: lwlock.c:1932
#define DSA_NUM_SIZE_CLASSES
Definition: dsa.c:247
#define DSA_FULLNESS_CLASSES
Definition: dsa.c:277
dsa_pointer start
Definition: dsa.c:199
uint64 dsa_pointer
Definition: dsa.h:62
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)
Definition: dsa.c:1525
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
Definition: dsa.c:1397
dsa_area_control * control
Definition: dsa.c:357
static const uint16 dsa_size_classes[]
Definition: dsa.c:236
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
#define Assert(condition)
Definition: c.h:804
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
Definition: dsa.c:309
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81
uint16 nallocatable
Definition: dsa.c:203
uint16 firstfree
Definition: dsa.c:204
#define NextFreeObjectIndex(object)
Definition: dsa.c:213

◆ attach_internal()

static dsa_area * attach_internal ( void *  place,
dsm_segment segment,
dsa_handle  handle 
)
static

Definition at line 1291 of file dsa.c.

References Assert, dsa_area::control, DSA_AREA_LOCK, DSA_MAX_SEGMENTS, DSA_SEGMENT_HEADER_MAGIC, ereport, errcode(), errmsg(), ERROR, dsa_segment_map::fpm, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area::mapping_pinned, MAXALIGN, dsa_segment_map::pagemap, palloc(), dsa_area_control::refcnt, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area_control::segment_header, and dsa_area::segment_maps.

Referenced by dsa_attach(), and dsa_attach_in_place().

1292 {
1293  dsa_area_control *control;
1294  dsa_area *area;
1295  dsa_segment_map *segment_map;
1296 
1297  control = (dsa_area_control *) place;
1298  Assert(control->handle == handle);
1299  Assert(control->segment_handles[0] == handle);
1300  Assert(control->segment_header.magic ==
1301  (DSA_SEGMENT_HEADER_MAGIC ^ handle ^ 0));
1302 
1303  /* Build the backend-local area object. */
1304  area = palloc(sizeof(dsa_area));
1305  area->control = control;
1306  area->mapping_pinned = false;
1307  memset(&area->segment_maps[0], 0,
1308  sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
1309  area->high_segment_index = 0;
1310 
1311  /* Set up the segment map for this process's mapping. */
1312  segment_map = &area->segment_maps[0];
1313  segment_map->segment = segment; /* NULL for in-place */
1314  segment_map->mapped_address = place;
1315  segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
1316  segment_map->fpm = (FreePageManager *)
1317  (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)));
1318  segment_map->pagemap = (dsa_pointer *)
1319  (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)) +
1320  MAXALIGN(sizeof(FreePageManager)));
1321 
1322  /* Bump the reference count. */
1324  if (control->refcnt == 0)
1325  {
1326  /* We can't attach to a DSA area that has already been destroyed. */
1327  ereport(ERROR,
1328  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1329  errmsg("could not attach to dynamic shared area")));
1330  }
1331  ++control->refcnt;
1334 
1335  return area;
1336 }
bool mapping_pinned
Definition: dsa.c:360
dsa_segment_index high_segment_index
Definition: dsa.c:371
dsm_segment * segment
Definition: dsa.c:341
int errcode(int sqlerrcode)
Definition: elog.c:698
uint64 dsa_pointer
Definition: dsa.h:62
FreePageManager * fpm
Definition: dsa.c:344
dsa_segment_header * header
Definition: dsa.c:343
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
int refcnt
Definition: dsa.c:317
#define ERROR
Definition: elog.h:46
dsa_area_control * control
Definition: dsa.c:357
dsa_segment_header segment_header
Definition: dsa.c:301
#define ereport(elevel,...)
Definition: elog.h:157
size_t freed_segment_counter
Definition: dsa.c:321
#define Assert(condition)
Definition: c.h:804
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:305
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
#define MAXALIGN(LEN)
Definition: c.h:757
size_t freed_segment_counter
Definition: dsa.c:374
dsa_pointer * pagemap
Definition: dsa.c:345
char * mapped_address
Definition: dsa.c:342
#define DSA_MAX_SEGMENTS
Definition: dsa.c:94
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
Definition: dsa.c:354
void * palloc(Size size)
Definition: mcxt.c:1062
int errmsg(const char *fmt,...)
Definition: elog.c:909
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
#define DSA_SEGMENT_HEADER_MAGIC
Definition: dsa.c:111
uint32 magic
Definition: dsa.c:154
dsa_handle handle
Definition: dsa.c:303

◆ check_for_freed_segments()

static void check_for_freed_segments ( dsa_area area)
static

Definition at line 2229 of file dsa.c.

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), pg_read_barrier, and unlikely.

Referenced by dsa_free(), and dsa_get_address().

2230 {
2231  size_t freed_segment_counter;
2232 
2233  /*
2234  * Any other process that has freed a segment has incremented
2235  * freed_segment_counter while holding an LWLock, and that must precede
2236  * any backend creating a new segment in the same slot while holding an
2237  * LWLock, and that must precede the creation of any dsa_pointer pointing
2238  * into the new segment which might reach us here, and the caller must
2239  * have sent the dsa_pointer to this process using appropriate memory
2240  * synchronization (some kind of locking or atomic primitive or system
2241  * call). So all we need to do on the reading side is ask for the load of
2242  * freed_segment_counter to follow the caller's load of the dsa_pointer it
2243  * has, and we can be sure to detect any segments that had been freed as
2244  * of the time that the dsa_pointer reached this process.
2245  */
2246  pg_read_barrier();
2247  freed_segment_counter = area->control->freed_segment_counter;
2248  if (unlikely(area->freed_segment_counter != freed_segment_counter))
2249  {
2250  /* Check all currently mapped segments to find what's been freed. */
2254  }
2255 }
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
dsa_area_control * control
Definition: dsa.c:357
size_t freed_segment_counter
Definition: dsa.c:321
#define pg_read_barrier()
Definition: atomics.h:158
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
size_t freed_segment_counter
Definition: dsa.c:374
#define unlikely(x)
Definition: c.h:273
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
static void check_for_freed_segments_locked(dsa_area *area)
Definition: dsa.c:2265

◆ check_for_freed_segments_locked()

static void check_for_freed_segments_locked ( dsa_area area)
static

Definition at line 2265 of file dsa.c.

References Assert, dsa_area::control, DSA_AREA_LOCK, dsm_detach(), dsa_segment_header::freed, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_segment_map::header, dsa_area::high_segment_index, i, LWLockHeldByMe(), dsa_segment_map::mapped_address, dsa_segment_map::segment, dsa_area::segment_maps, and unlikely.

Referenced by check_for_freed_segments(), destroy_superblock(), dsa_dump(), and get_best_segment().

2266 {
2267  size_t freed_segment_counter;
2268  int i;
2269 
2271  freed_segment_counter = area->control->freed_segment_counter;
2272  if (unlikely(area->freed_segment_counter != freed_segment_counter))
2273  {
2274  for (i = 0; i <= area->high_segment_index; ++i)
2275  {
2276  if (area->segment_maps[i].header != NULL &&
2277  area->segment_maps[i].header->freed)
2278  {
2279  dsm_detach(area->segment_maps[i].segment);
2280  area->segment_maps[i].segment = NULL;
2281  area->segment_maps[i].header = NULL;
2282  area->segment_maps[i].mapped_address = NULL;
2283  }
2284  }
2285  area->freed_segment_counter = freed_segment_counter;
2286  }
2287 }
bool LWLockHeldByMe(LWLock *l)
Definition: lwlock.c:1932
dsa_segment_index high_segment_index
Definition: dsa.c:371
dsm_segment * segment
Definition: dsa.c:341
dsa_segment_header * header
Definition: dsa.c:343
dsa_area_control * control
Definition: dsa.c:357
size_t freed_segment_counter
Definition: dsa.c:321
#define Assert(condition)
Definition: c.h:804
size_t freed_segment_counter
Definition: dsa.c:374
char * mapped_address
Definition: dsa.c:342
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
void dsm_detach(dsm_segment *seg)
Definition: dsm.c:769
int i
#define unlikely(x)
Definition: c.h:273
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143

◆ create_internal()

static dsa_area * create_internal ( void *  place,
size_t  size,
int  tranche_id,
dsm_handle  control_handle,
dsm_segment control_segment 
)
static

Definition at line 1191 of file dsa.c.

References Assert, dsa_segment_header::bin, contiguous_pages_to_segment_bin, dsa_area::control, DSA_INITIAL_SEGMENT_SIZE, DSA_MAX_SEGMENTS, dsa_minimum_size(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, elog, ERROR, dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area::freed_segment_counter, FreePageManagerInitialize(), FreePageManagerPut(), dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, i, dsa_area_control::lock, dsa_area_control::lwlock_tranche_id, LWLockInitialize(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area::mapping_pinned, dsa_area_control::max_total_segment_size, MAXALIGN, dsa_segment_header::next, dsa_segment_map::pagemap, palloc(), dsa_segment_header::prev, dsa_area_control::refcnt, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area_control::segment_header, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

Referenced by dsa_create(), and dsa_create_in_place().

1195 {
1196  dsa_area_control *control;
1197  dsa_area *area;
1198  dsa_segment_map *segment_map;
1199  size_t usable_pages;
1200  size_t total_pages;
1201  size_t metadata_bytes;
1202  int i;
1203 
1204  /* Sanity check on the space we have to work in. */
1205  if (size < dsa_minimum_size())
1206  elog(ERROR, "dsa_area space must be at least %zu, but %zu provided",
1207  dsa_minimum_size(), size);
1208 
1209  /* Now figure out how much space is usable */
1210  total_pages = size / FPM_PAGE_SIZE;
1211  metadata_bytes =
1212  MAXALIGN(sizeof(dsa_area_control)) +
1213  MAXALIGN(sizeof(FreePageManager)) +
1214  total_pages * sizeof(dsa_pointer);
1215  /* Add padding up to next page boundary. */
1216  if (metadata_bytes % FPM_PAGE_SIZE != 0)
1217  metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
1218  Assert(metadata_bytes <= size);
1219  usable_pages = (size - metadata_bytes) / FPM_PAGE_SIZE;
1220 
1221  /*
1222  * Initialize the dsa_area_control object located at the start of the
1223  * space.
1224  */
1225  control = (dsa_area_control *) place;
1226  memset(place, 0, sizeof(*control));
1227  control->segment_header.magic =
1228  DSA_SEGMENT_HEADER_MAGIC ^ control_handle ^ 0;
1231  control->segment_header.usable_pages = usable_pages;
1232  control->segment_header.freed = false;
1234  control->handle = control_handle;
1235  control->max_total_segment_size = (size_t) -1;
1236  control->total_segment_size = size;
1237  control->segment_handles[0] = control_handle;
1238  for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
1239  control->segment_bins[i] = DSA_SEGMENT_INDEX_NONE;
1240  control->refcnt = 1;
1241  control->lwlock_tranche_id = tranche_id;
1242 
1243  /*
1244  * Create the dsa_area object that this backend will use to access the
1245  * area. Other backends will need to obtain their own dsa_area object by
1246  * attaching.
1247  */
1248  area = palloc(sizeof(dsa_area));
1249  area->control = control;
1250  area->mapping_pinned = false;
1251  memset(area->segment_maps, 0, sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
1252  area->high_segment_index = 0;
1253  area->freed_segment_counter = 0;
1254  LWLockInitialize(&control->lock, control->lwlock_tranche_id);
1255  for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
1257  control->lwlock_tranche_id);
1258 
1259  /* Set up the segment map for this process's mapping. */
1260  segment_map = &area->segment_maps[0];
1261  segment_map->segment = control_segment;
1262  segment_map->mapped_address = place;
1263  segment_map->header = (dsa_segment_header *) place;
1264  segment_map->fpm = (FreePageManager *)
1265  (segment_map->mapped_address +
1266  MAXALIGN(sizeof(dsa_area_control)));
1267  segment_map->pagemap = (dsa_pointer *)
1268  (segment_map->mapped_address +
1269  MAXALIGN(sizeof(dsa_area_control)) +
1270  MAXALIGN(sizeof(FreePageManager)));
1271 
1272  /* Set up the free page map. */
1273  FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
1274  /* There can be 0 usable pages if size is dsa_minimum_size(). */
1275 
1276  if (usable_pages > 0)
1277  FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
1278  usable_pages);
1279 
1280  /* Put this segment into the appropriate bin. */
1281  control->segment_bins[contiguous_pages_to_segment_bin(usable_pages)] = 0;
1282  segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
1283 
1284  return area;
1285 }
bool mapping_pinned
Definition: dsa.c:360
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
Definition: dsa.c:307
#define contiguous_pages_to_segment_bin(n)
Definition: dsa.c:140
size_t usable_pages
Definition: dsa.c:156
#define DSA_NUM_SIZE_CLASSES
Definition: dsa.c:247
dsa_segment_index high_segment_index
Definition: dsa.c:371
dsm_segment * segment
Definition: dsa.c:341
size_t size
Definition: dsa.c:158
dsa_segment_index prev
Definition: dsa.c:164
size_t dsa_minimum_size(void)
Definition: dsa.c:1169
int lwlock_tranche_id
Definition: dsa.c:323
size_t total_segment_size
Definition: dsa.c:311
uint64 dsa_pointer
Definition: dsa.h:62
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
Definition: freepage.c:379
FreePageManager * fpm
Definition: dsa.c:344
dsa_segment_header * header
Definition: dsa.c:343
LWLock lock
Definition: dsa.c:325
int refcnt
Definition: dsa.c:317
#define DSA_INITIAL_SEGMENT_SIZE
Definition: dsa.c:68
#define ERROR
Definition: elog.h:46
dsa_segment_index next
Definition: dsa.c:170
dsa_area_control * control
Definition: dsa.c:357
#define DSA_NUM_SEGMENT_BINS
Definition: dsa.c:133
size_t max_total_segment_size
Definition: dsa.c:313
void LWLockInitialize(LWLock *lock, int tranche_id)
Definition: lwlock.c:740
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
Definition: freepage.c:183
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
dsa_segment_header segment_header
Definition: dsa.c:301
#define FPM_PAGE_SIZE
Definition: freepage.h:30
#define Assert(condition)
Definition: c.h:804
#define DSA_SEGMENT_INDEX_NONE
Definition: dsa.c:127
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:305
#define MAXALIGN(LEN)
Definition: c.h:757
size_t freed_segment_counter
Definition: dsa.c:374
dsa_pointer * pagemap
Definition: dsa.c:345
char * mapped_address
Definition: dsa.c:342
#define DSA_MAX_SEGMENTS
Definition: dsa.c:94
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
Definition: dsa.c:354
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
int i
#define DSA_SEGMENT_HEADER_MAGIC
Definition: dsa.c:111
uint32 magic
Definition: dsa.c:154
size_t bin
Definition: dsa.c:172
dsa_handle handle
Definition: dsa.c:303

◆ destroy_superblock()

static void destroy_superblock ( dsa_area area,
dsa_pointer  span_pointer 
)
static

Definition at line 1800 of file dsa.c.

References Assert, check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, dsm_detach(), DSM_HANDLE_INVALID, dsm_segment_handle(), dsm_unpin_segment(), dsa_segment_map::fpm, fpm_largest, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area_control::freed_segment_counter, FreePageManagerPut(), get_segment_by_index(), get_segment_index, dsa_segment_map::header, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_span::npages, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_segment_header::size, dsa_area_span::size_class, dsa_area_span::start, dsa_area_control::total_segment_size, unlink_segment(), unlink_span(), and dsa_segment_header::usable_pages.

Referenced by dsa_free(), and dsa_trim().

1801 {
1802  dsa_area_span *span = dsa_get_address(area, span_pointer);
1803  int size_class = span->size_class;
1804  dsa_segment_map *segment_map;
1805 
1806 
1807  /* Remove it from its fullness class list. */
1808  unlink_span(area, span);
1809 
1810  /*
1811  * Note: Here we acquire the area lock while we already hold a per-pool
1812  * lock. We never hold the area lock and then take a pool lock, or we
1813  * could deadlock.
1814  */
1817  segment_map =
1819  FreePageManagerPut(segment_map->fpm,
1821  span->npages);
1822  /* Check if the segment is now entirely free. */
1823  if (fpm_largest(segment_map->fpm) == segment_map->header->usable_pages)
1824  {
1825  dsa_segment_index index = get_segment_index(area, segment_map);
1826 
1827  /* If it's not the segment with extra control data, free it. */
1828  if (index != 0)
1829  {
1830  /*
1831  * Give it back to the OS, and allow other backends to detect that
1832  * they need to detach.
1833  */
1834  unlink_segment(area, segment_map);
1835  segment_map->header->freed = true;
1837  segment_map->header->size);
1838  area->control->total_segment_size -=
1839  segment_map->header->size;
1841  dsm_detach(segment_map->segment);
1842  area->control->segment_handles[index] = DSM_HANDLE_INVALID;
1843  ++area->control->freed_segment_counter;
1844  segment_map->segment = NULL;
1845  segment_map->header = NULL;
1846  segment_map->mapped_address = NULL;
1847  }
1848  }
1850 
1851  /*
1852  * Span-of-spans blocks store the span which describes them within the
1853  * block itself, so freeing the storage implicitly frees the descriptor
1854  * also. If this is a block of any other type, we need to separately free
1855  * the span object also. This recursive call to dsa_free will acquire the
1856  * span pool's lock. We can't deadlock because the acquisition order is
1857  * always some other pool and then the span pool.
1858  */
1859  if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
1860  dsa_free(area, span_pointer);
1861 }
size_t usable_pages
Definition: dsa.c:156
dsm_segment * segment
Definition: dsa.c:341
size_t size
Definition: dsa.c:158
dsm_handle dsm_segment_handle(dsm_segment *seg)
Definition: dsm.c:1087
dsa_pointer start
Definition: dsa.c:199
size_t total_segment_size
Definition: dsa.c:311
size_t npages
Definition: dsa.c:200
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
Definition: freepage.c:379
FreePageManager * fpm
Definition: dsa.c:344
dsa_segment_header * header
Definition: dsa.c:343
#define get_segment_index(area, segment_map_ptr)
Definition: dsa.c:381
Definition: type.h:89
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
uint16 size_class
Definition: dsa.c:201
dsa_area_control * control
Definition: dsa.c:357
void dsm_unpin_segment(dsm_handle handle)
Definition: dsm.c:953
static void unlink_span(dsa_area *area, dsa_area_span *span)
Definition: dsa.c:1864
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
Definition: dsa.c:1936
#define FPM_PAGE_SIZE
Definition: freepage.h:30
size_t freed_segment_counter
Definition: dsa.c:321
#define DSA_SCLASS_BLOCK_OF_SPANS
Definition: dsa.c:250
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
Definition: dsa.c:118
#define Assert(condition)
Definition: c.h:804
#define fpm_largest(fpm)
Definition: freepage.h:88
#define DSA_EXTRACT_OFFSET(dp)
Definition: dsa.c:121
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:305
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
size_t dsa_segment_index
Definition: dsa.c:124
char * mapped_address
Definition: dsa.c:342
void dsa_free(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:820
void dsm_detach(dsm_segment *seg)
Definition: dsm.c:769
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
static void check_for_freed_segments_locked(dsa_area *area)
Definition: dsa.c:2265
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ dsa_allocate_extended()

dsa_pointer dsa_allocate_extended ( dsa_area area,
size_t  size,
int  flags 
)

Definition at line 665 of file dsa.c.

References alloc_object(), AllocHugeSizeIsValid, AllocSizeIsValid, Assert, dsa_area::control, DSA_ALLOC_HUGE, DSA_ALLOC_NO_OOM, DSA_ALLOC_ZERO, DSA_AREA_LOCK, dsa_free(), dsa_get_address(), DSA_MAKE_POINTER, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_class_map, DSA_SIZE_CLASS_MAP_QUANTUM, dsa_size_classes, DsaPointerIsValid, elog, ereport, errcode(), errdetail(), errmsg(), ERROR, FATAL, dsa_segment_map::fpm, FPM_PAGE_SIZE, fpm_size_to_pages, FreePageManagerGet(), get_best_segment(), get_segment_index, init_span(), InvalidDsaPointer, lengthof, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), make_new_segment(), dsa_segment_map::pagemap, and dsa_area_control::pools.

Referenced by dshash_create(), and pagetable_allocate().

666 {
667  uint16 size_class;
668  dsa_pointer start_pointer;
669  dsa_segment_map *segment_map;
670  dsa_pointer result;
671 
672  Assert(size > 0);
673 
674  /* Sanity check on huge individual allocation size. */
675  if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
676  ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
677  elog(ERROR, "invalid DSA memory alloc request size %zu", size);
678 
679  /*
680  * If bigger than the largest size class, just grab a run of pages from
681  * the free page manager, instead of allocating an object from a pool.
682  * There will still be a span, but it's a special class of span that
683  * manages this whole allocation and simply gives all pages back to the
684  * free page manager when dsa_free is called.
685  */
686  if (size > dsa_size_classes[lengthof(dsa_size_classes) - 1])
687  {
688  size_t npages = fpm_size_to_pages(size);
689  size_t first_page;
690  dsa_pointer span_pointer;
692 
693  /* Obtain a span object. */
694  span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
695  if (!DsaPointerIsValid(span_pointer))
696  {
697  /* Raise error unless asked not to. */
698  if ((flags & DSA_ALLOC_NO_OOM) == 0)
699  ereport(ERROR,
700  (errcode(ERRCODE_OUT_OF_MEMORY),
701  errmsg("out of memory"),
702  errdetail("Failed on DSA request of size %zu.",
703  size)));
704  return InvalidDsaPointer;
705  }
706 
708 
709  /* Find a segment from which to allocate. */
710  segment_map = get_best_segment(area, npages);
711  if (segment_map == NULL)
712  segment_map = make_new_segment(area, npages);
713  if (segment_map == NULL)
714  {
715  /* Can't make any more segments: game over. */
717  dsa_free(area, span_pointer);
718 
719  /* Raise error unless asked not to. */
720  if ((flags & DSA_ALLOC_NO_OOM) == 0)
721  ereport(ERROR,
722  (errcode(ERRCODE_OUT_OF_MEMORY),
723  errmsg("out of memory"),
724  errdetail("Failed on DSA request of size %zu.",
725  size)));
726  return InvalidDsaPointer;
727  }
728 
729  /*
730  * Ask the free page manager for a run of pages. This should always
731  * succeed, since both get_best_segment and make_new_segment should
732  * only return a non-NULL pointer if it actually contains enough
733  * contiguous freespace. If it does fail, something in our backend
734  * private state is out of whack, so use FATAL to kill the process.
735  */
736  if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
737  elog(FATAL,
738  "dsa_allocate could not find %zu free pages", npages);
740 
741  start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
742  first_page * FPM_PAGE_SIZE);
743 
744  /* Initialize span and pagemap. */
746  LW_EXCLUSIVE);
747  init_span(area, span_pointer, pool, start_pointer, npages,
749  segment_map->pagemap[first_page] = span_pointer;
751 
752  /* Zero-initialize the memory if requested. */
753  if ((flags & DSA_ALLOC_ZERO) != 0)
754  memset(dsa_get_address(area, start_pointer), 0, size);
755 
756  return start_pointer;
757  }
758 
759  /* Map allocation to a size class. */
761  {
762  int mapidx;
763 
764  /* For smaller sizes we have a lookup table... */
765  mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
766  DSA_SIZE_CLASS_MAP_QUANTUM) - 1;
767  size_class = dsa_size_class_map[mapidx];
768  }
769  else
770  {
771  uint16 min;
772  uint16 max;
773 
774  /* ... and for the rest we search by binary chop. */
776  max = lengthof(dsa_size_classes) - 1;
777 
778  while (min < max)
779  {
780  uint16 mid = (min + max) / 2;
781  uint16 class_size = dsa_size_classes[mid];
782 
783  if (class_size < size)
784  min = mid + 1;
785  else
786  max = mid;
787  }
788 
789  size_class = min;
790  }
791  Assert(size <= dsa_size_classes[size_class]);
792  Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
793 
794  /* Attempt to allocate an object from the appropriate pool. */
795  result = alloc_object(area, size_class);
796 
797  /* Check for failure to allocate. */
798  if (!DsaPointerIsValid(result))
799  {
800  /* Raise error unless asked not to. */
801  if ((flags & DSA_ALLOC_NO_OOM) == 0)
802  ereport(ERROR,
803  (errcode(ERRCODE_OUT_OF_MEMORY),
804  errmsg("out of memory"),
805  errdetail("Failed on DSA request of size %zu.", size)));
806  return InvalidDsaPointer;
807  }
808 
809  /* Zero-initialize the memory if requested. */
810  if ((flags & DSA_ALLOC_ZERO) != 0)
811  memset(dsa_get_address(area, result), 0, size);
812 
813  return result;
814 }
#define DSA_ALLOC_NO_OOM
Definition: dsa.h:74
#define InvalidDsaPointer
Definition: dsa.h:78
#define DSA_ALLOC_ZERO
Definition: dsa.h:75
#define fpm_size_to_pages(sz)
Definition: freepage.h:74
int errcode(int sqlerrcode)
Definition: elog.c:698
#define lengthof(array)
Definition: c.h:734
uint64 dsa_pointer
Definition: dsa.h:62
FreePageManager * fpm
Definition: dsa.c:344
#define get_segment_index(area, segment_map_ptr)
Definition: dsa.c:381
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
Definition: freepage.c:210
unsigned short uint16
Definition: c.h:440
#define ERROR
Definition: elog.h:46
#define FATAL
Definition: elog.h:49
dsa_area_control * control
Definition: dsa.c:357
int errdetail(const char *fmt,...)
Definition: elog.c:1042
static const uint16 dsa_size_classes[]
Definition: dsa.c:236
#define AllocSizeIsValid(size)
Definition: memutils.h:42
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
#define DSA_SIZE_CLASS_MAP_QUANTUM
Definition: dsa.c:269
#define ereport(elevel,...)
Definition: elog.h:157
#define FPM_PAGE_SIZE
Definition: freepage.h:30
#define DSA_SCLASS_BLOCK_OF_SPANS
Definition: dsa.c:250
#define DSA_ALLOC_HUGE
Definition: dsa.h:73
#define Assert(condition)
Definition: c.h:804
#define DSA_MAKE_POINTER(segment_number, offset)
Definition: dsa.c:114
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
Definition: dsa.c:309
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
dsa_pointer * pagemap
Definition: dsa.c:345
#define AllocHugeSizeIsValid(size)
Definition: memutils.h:46
static const uint8 dsa_size_class_map[]
Definition: dsa.c:259
#define DsaPointerIsValid(x)
Definition: dsa.h:81
void dsa_free(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:820
int errmsg(const char *fmt,...)
Definition: elog.c:909
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
Definition: dsa.c:2060
#define elog(elevel,...)
Definition: elog.h:232
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
Definition: dsa.c:1342
static dsa_pointer alloc_object(dsa_area *area, int size_class)
Definition: dsa.c:1437
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
#define DSA_SCLASS_SPAN_LARGE
Definition: dsa.c:251
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
Definition: dsa.c:1968

◆ dsa_attach()

dsa_area* dsa_attach ( dsa_handle  handle)

Definition at line 506 of file dsa.c.

References attach_internal(), dsa_on_dsm_detach_release_in_place(), dsm_attach(), dsm_segment_address(), ereport, errcode(), errmsg(), ERROR, on_dsm_detach(), and PointerGetDatum.

507 {
508  dsm_segment *segment;
509  dsa_area *area;
510 
511  /*
512  * An area handle is really a DSM segment handle for the first segment, so
513  * we go ahead and attach to that.
514  */
515  segment = dsm_attach(handle);
516  if (segment == NULL)
517  ereport(ERROR,
518  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
519  errmsg("could not attach to dynamic shared area")));
520 
521  area = attach_internal(dsm_segment_address(segment), segment, handle);
522 
523  /* Clean up when the control segment detaches. */
526 
527  return area;
528 }
#define PointerGetDatum(X)
Definition: postgres.h:600
dsm_segment * dsm_attach(dsm_handle h)
Definition: dsm.c:631
int errcode(int sqlerrcode)
Definition: elog.c:698
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
Definition: dsa.c:1291
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
Definition: dsm.c:1096
#define ERROR
Definition: elog.h:46
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
Definition: dsa.c:572
#define ereport(elevel,...)
Definition: elog.h:157
void * dsm_segment_address(dsm_segment *seg)
Definition: dsm.c:1059
Definition: dsa.c:354
int errmsg(const char *fmt,...)
Definition: elog.c:909

◆ dsa_attach_in_place()

dsa_area* dsa_attach_in_place ( void *  place,
dsm_segment segment 
)

Definition at line 541 of file dsa.c.

References attach_internal(), dsa_on_dsm_detach_release_in_place(), DSM_HANDLE_INVALID, on_dsm_detach(), and PointerGetDatum.

Referenced by AttachSession(), and ParallelQueryMain().

542 {
543  dsa_area *area;
544 
545  area = attach_internal(place, NULL, DSM_HANDLE_INVALID);
546 
547  /*
548  * Clean up when the control segment detaches, if a containing DSM segment
549  * was provided.
550  */
551  if (segment != NULL)
553  PointerGetDatum(place));
554 
555  return area;
556 }
#define PointerGetDatum(X)
Definition: postgres.h:600
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
Definition: dsa.c:1291
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
Definition: dsm.c:1096
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
Definition: dsa.c:572
Definition: dsa.c:354

◆ dsa_create()

dsa_area* dsa_create ( int  tranche_id)

Definition at line 420 of file dsa.c.

References create_internal(), DSA_INITIAL_SEGMENT_SIZE, dsa_on_dsm_detach_release_in_place(), dsm_create(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), on_dsm_detach(), and PointerGetDatum.

421 {
422  dsm_segment *segment;
423  dsa_area *area;
424 
425  /*
426  * Create the DSM segment that will hold the shared control object and the
427  * first segment of usable space.
428  */
429  segment = dsm_create(DSA_INITIAL_SEGMENT_SIZE, 0);
430 
431  /*
432  * All segments backing this area are pinned, so that DSA can explicitly
433  * control their lifetime (otherwise a newly created segment belonging to
434  * this area might be freed when the only backend that happens to have it
435  * mapped in ends, corrupting the area).
436  */
437  dsm_pin_segment(segment);
438 
439  /* Create a new DSA area with the control object in this segment. */
440  area = create_internal(dsm_segment_address(segment),
442  tranche_id,
443  dsm_segment_handle(segment), segment);
444 
445  /* Clean up when the control segment detaches. */
448 
449  return area;
450 }
#define PointerGetDatum(X)
Definition: postgres.h:600
dsm_handle dsm_segment_handle(dsm_segment *seg)
Definition: dsm.c:1087
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
Definition: dsm.c:1096
void dsm_pin_segment(dsm_segment *seg)
Definition: dsm.c:921
#define DSA_INITIAL_SEGMENT_SIZE
Definition: dsa.c:68
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment)
Definition: dsa.c:1191
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
Definition: dsa.c:572
dsm_segment * dsm_create(Size size, int flags)
Definition: dsm.c:487
void * dsm_segment_address(dsm_segment *seg)
Definition: dsm.c:1059
Definition: dsa.c:354

◆ dsa_create_in_place()

dsa_area* dsa_create_in_place ( void *  place,
size_t  size,
int  tranche_id,
dsm_segment segment 
)

Definition at line 469 of file dsa.c.

References create_internal(), dsa_on_dsm_detach_release_in_place(), DSM_HANDLE_INVALID, on_dsm_detach(), and PointerGetDatum.

Referenced by ExecInitParallelPlan(), and GetSessionDsmHandle().

471 {
472  dsa_area *area;
473 
474  area = create_internal(place, size, tranche_id,
475  DSM_HANDLE_INVALID, NULL);
476 
477  /*
478  * Clean up when the control segment detaches, if a containing DSM segment
479  * was provided.
480  */
481  if (segment != NULL)
483  PointerGetDatum(place));
484 
485  return area;
486 }
#define PointerGetDatum(X)
Definition: postgres.h:600
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
Definition: dsm.c:1096
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment)
Definition: dsa.c:1191
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
Definition: dsa.c:572
Definition: dsa.c:354

◆ dsa_detach()

void dsa_detach ( dsa_area area)

Definition at line 1910 of file dsa.c.

References dsm_detach(), dsa_area::high_segment_index, i, pfree(), dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by DetachSession(), ExecParallelCleanup(), and ParallelQueryMain().

1911 {
1912  int i;
1913 
1914  /* Detach from all segments. */
1915  for (i = 0; i <= area->high_segment_index; ++i)
1916  if (area->segment_maps[i].segment != NULL)
1917  dsm_detach(area->segment_maps[i].segment);
1918 
1919  /*
1920  * Note that 'detaching' (= detaching from DSM segments) doesn't include
1921  * 'releasing' (= adjusting the reference count). It would be nice to
1922  * combine these operations, but client code might never get around to
1923  * calling dsa_detach because of an error path, and a detach hook on any
1924  * particular segment is too late to detach other segments in the area
1925  * without risking a 'leak' warning in the non-error path.
1926  */
1927 
1928  /* Free the backend-local area object. */
1929  pfree(area);
1930 }
dsa_segment_index high_segment_index
Definition: dsa.c:371
dsm_segment * segment
Definition: dsa.c:341
void pfree(void *pointer)
Definition: mcxt.c:1169
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
void dsm_detach(dsm_segment *seg)
Definition: dsm.c:769
int i

◆ dsa_dump()

void dsa_dump ( dsa_area area)

Definition at line 1065 of file dsa.c.

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_POINTER_FORMAT, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DSA_SEGMENT_INDEX_NONE, dsa_size_classes, DsaPointerIsValid, dsa_segment_map::fpm, fpm_largest, fprintf, get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, i, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_control::max_total_segment_size, dsa_area_span::nallocatable, dsa_segment_header::next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_control::pinned, dsa_area_control::pools, dsa_area_control::refcnt, dsa_area_control::segment_bins, dsa_area_pool::spans, dsa_area_span::start, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

1066 {
1067  size_t i,
1068  j;
1069 
1070  /*
1071  * Note: This gives an inconsistent snapshot as it acquires and releases
1072  * individual locks as it goes...
1073  */
1074 
1077  fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
1078  fprintf(stderr, " max_total_segment_size: %zu\n",
1080  fprintf(stderr, " total_segment_size: %zu\n",
1081  area->control->total_segment_size);
1082  fprintf(stderr, " refcnt: %d\n", area->control->refcnt);
1083  fprintf(stderr, " pinned: %c\n", area->control->pinned ? 't' : 'f');
1084  fprintf(stderr, " segment bins:\n");
1085  for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
1086  {
1087  if (area->control->segment_bins[i] != DSA_SEGMENT_INDEX_NONE)
1088  {
1089  dsa_segment_index segment_index;
1090 
1091  fprintf(stderr,
1092  " segment bin %zu (at least %d contiguous pages free):\n",
1093  i, 1 << (i - 1));
1094  segment_index = area->control->segment_bins[i];
1095  while (segment_index != DSA_SEGMENT_INDEX_NONE)
1096  {
1097  dsa_segment_map *segment_map;
1098 
1099  segment_map =
1100  get_segment_by_index(area, segment_index);
1101 
1102  fprintf(stderr,
1103  " segment index %zu, usable_pages = %zu, "
1104  "contiguous_pages = %zu, mapped at %p\n",
1105  segment_index,
1106  segment_map->header->usable_pages,
1107  fpm_largest(segment_map->fpm),
1108  segment_map->mapped_address);
1109  segment_index = segment_map->header->next;
1110  }
1111  }
1112  }
1114 
1115  fprintf(stderr, " pools:\n");
1116  for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
1117  {
1118  bool found = false;
1119 
1121  for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
1122  if (DsaPointerIsValid(area->control->pools[i].spans[j]))
1123  found = true;
1124  if (found)
1125  {
1126  if (i == DSA_SCLASS_BLOCK_OF_SPANS)
1127  fprintf(stderr, " pool for blocks of span objects:\n");
1128  else if (i == DSA_SCLASS_SPAN_LARGE)
1129  fprintf(stderr, " pool for large object spans:\n");
1130  else
1131  fprintf(stderr,
1132  " pool for size class %zu (object size %hu bytes):\n",
1133  i, dsa_size_classes[i]);
1134  for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
1135  {
1136  if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
1137  fprintf(stderr, " fullness class %zu is empty\n", j);
1138  else
1139  {
1140  dsa_pointer span_pointer = area->control->pools[i].spans[j];
1141 
1142  fprintf(stderr, " fullness class %zu:\n", j);
1143  while (DsaPointerIsValid(span_pointer))
1144  {
1145  dsa_area_span *span;
1146 
1147  span = dsa_get_address(area, span_pointer);
1148  fprintf(stderr,
1149  " span descriptor at "
1150  DSA_POINTER_FORMAT ", superblock at "
1152  ", pages = %zu, objects free = %hu/%hu\n",
1153  span_pointer, span->start, span->npages,
1154  span->nallocatable, span->nmax);
1155  span_pointer = span->nextspan;
1156  }
1157  }
1158  }
1159  }
1160  LWLockRelease(DSA_SCLASS_LOCK(area, i));
1161  }
1162 }
uint16 nmax
Definition: dsa.c:205
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
Definition: dsa.c:307
dsa_pointer nextspan
Definition: dsa.c:198
size_t usable_pages
Definition: dsa.c:156
#define DSA_NUM_SIZE_CLASSES
Definition: dsa.c:247
bool pinned
Definition: dsa.c:319
#define DSA_FULLNESS_CLASSES
Definition: dsa.c:277
dsa_pointer start
Definition: dsa.c:199
#define fprintf
Definition: port.h:220
size_t total_segment_size
Definition: dsa.c:311
size_t npages
Definition: dsa.c:200
uint64 dsa_pointer
Definition: dsa.h:62
FreePageManager * fpm
Definition: dsa.c:344
dsa_segment_header * header
Definition: dsa.c:343
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
#define DSA_POINTER_FORMAT
Definition: dsa.h:69
int refcnt
Definition: dsa.c:317
dsa_segment_index next
Definition: dsa.c:170
dsa_area_control * control
Definition: dsa.c:357
#define DSA_NUM_SEGMENT_BINS
Definition: dsa.c:133
static const uint16 dsa_size_classes[]
Definition: dsa.c:236
size_t max_total_segment_size
Definition: dsa.c:313
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
#define DSA_SCLASS_BLOCK_OF_SPANS
Definition: dsa.c:250
#define fpm_largest(fpm)
Definition: freepage.h:88
#define DSA_SEGMENT_INDEX_NONE
Definition: dsa.c:127
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
Definition: dsa.c:309
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
size_t dsa_segment_index
Definition: dsa.c:124
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
char * mapped_address
Definition: dsa.c:342
#define DsaPointerIsValid(x)
Definition: dsa.h:81
uint16 nallocatable
Definition: dsa.c:203
int i
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
#define DSA_SCLASS_SPAN_LARGE
Definition: dsa.c:251
dsa_handle handle
Definition: dsa.c:303
static void check_for_freed_segments_locked(dsa_area *area)
Definition: dsa.c:2265
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ dsa_free()

void dsa_free ( dsa_area area,
dsa_pointer  dp 
)

Definition at line 820 of file dsa.c.

References add_span_to_fullness_class(), Assert, check_for_freed_segments(), destroy_superblock(), DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SUPERBLOCK_SIZE, dsa_area_span::fclass, dsa_area_span::firstfree, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerPut(), get_segment_by_index(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::nmax, dsa_area_span::npages, dsa_segment_map::pagemap, dsa_area_span::prevspan, dsa_area_span::size_class, dsa_area_span::start, and unlink_span().

Referenced by delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_free(), dshash_create(), dshash_destroy(), ExecHashTableDetach(), ExecHashTableDetachBatch(), ExecParallelCleanup(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashRepartitionFirst(), ExecParallelReinitialize(), find_or_make_matching_shared_tupledesc(), pagetable_free(), resize(), and tbm_free_shared_area().

821 {
822  dsa_segment_map *segment_map;
823  int pageno;
824  dsa_pointer span_pointer;
825  dsa_area_span *span;
826  char *superblock;
827  char *object;
828  size_t size;
829  int size_class;
830 
831  /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
833 
834  /* Locate the object, span and pool. */
835  segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
836  pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
837  span_pointer = segment_map->pagemap[pageno];
838  span = dsa_get_address(area, span_pointer);
839  superblock = dsa_get_address(area, span->start);
840  object = dsa_get_address(area, dp);
841  size_class = span->size_class;
842  size = dsa_size_classes[size_class];
843 
844  /*
845  * Special case for large objects that live in a special span: we return
846  * those pages directly to the free page manager and free the span.
847  */
848  if (span->size_class == DSA_SCLASS_SPAN_LARGE)
849  {
850 
851 #ifdef CLOBBER_FREED_MEMORY
852  memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
853 #endif
854 
855  /* Give pages back to free page manager. */
857  FreePageManagerPut(segment_map->fpm,
859  span->npages);
861  /* Unlink span. */
863  LW_EXCLUSIVE);
864  unlink_span(area, span);
866  /* Free the span object so it can be reused. */
867  dsa_free(area, span_pointer);
868  return;
869  }
870 
871 #ifdef CLOBBER_FREED_MEMORY
872  memset(object, 0x7f, size);
873 #endif
874 
875  LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
876 
877  /* Put the object on the span's freelist. */
878  Assert(object >= superblock);
879  Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
880  Assert((object - superblock) % size == 0);
881  NextFreeObjectIndex(object) = span->firstfree;
882  span->firstfree = (object - superblock) / size;
883  ++span->nallocatable;
884 
885  /*
886  * See if the span needs to moved to a different fullness class, or be
887  * freed so its pages can be given back to the segment.
888  */
889  if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
890  {
891  /*
892  * The block was completely full and is located in the
893  * highest-numbered fullness class, which is never scanned for free
894  * chunks. We must move it to the next-lower fullness class.
895  */
896  unlink_span(area, span);
897  add_span_to_fullness_class(area, span, span_pointer,
899 
900  /*
901  * If this is the only span, and there is no active span, then we
902  * should probably move this span to fullness class 1. (Otherwise if
903  * you allocate exactly all the objects in the only span, it moves to
904  * class 3, then you free them all, it moves to 2, and then is given
905  * back, leaving no active span).
906  */
907  }
908  else if (span->nallocatable == span->nmax &&
909  (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
910  {
911  /*
912  * This entire block is free, and it's not the active block for this
913  * size class. Return the memory to the free page manager. We don't
914  * do this for the active block to prevent hysteresis: if we
915  * repeatedly allocate and free the only chunk in the active block, it
916  * will be very inefficient if we deallocate and reallocate the block
917  * every time.
918  */
919  destroy_superblock(area, span_pointer);
920  }
921 
922  LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
923 }
uint16 nmax
Definition: dsa.c:205
#define DSA_SUPERBLOCK_SIZE
Definition: dsa.c:378
#define InvalidDsaPointer
Definition: dsa.h:78
#define DSA_FULLNESS_CLASSES
Definition: dsa.c:277
dsa_pointer start
Definition: dsa.c:199
size_t npages
Definition: dsa.c:200
uint64 dsa_pointer
Definition: dsa.h:62
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
Definition: freepage.c:379
FreePageManager * fpm
Definition: dsa.c:344
uint16 fclass
Definition: dsa.c:206
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
uint16 size_class
Definition: dsa.c:201
static const uint16 dsa_size_classes[]
Definition: dsa.c:236
static void unlink_span(dsa_area *area, dsa_area_span *span)
Definition: dsa.c:1864
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
dsa_pointer prevspan
Definition: dsa.c:197
#define FPM_PAGE_SIZE
Definition: freepage.h:30
static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
Definition: dsa.c:1887
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
Definition: dsa.c:118
#define Assert(condition)
Definition: c.h:804
#define DSA_EXTRACT_OFFSET(dp)
Definition: dsa.c:121
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
dsa_pointer * pagemap
Definition: dsa.c:345
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
Definition: dsa.c:1800
void dsa_free(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:820
uint16 nallocatable
Definition: dsa.c:203
static void check_for_freed_segments(dsa_area *area)
Definition: dsa.c:2229
uint16 firstfree
Definition: dsa.c:204
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
#define DSA_SCLASS_SPAN_LARGE
Definition: dsa.c:251
#define NextFreeObjectIndex(object)
Definition: dsa.c:213
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ dsa_get_address()

void* dsa_get_address ( dsa_area area,
dsa_pointer  dp 
)

Definition at line 932 of file dsa.c.

References Assert, check_for_freed_segments(), DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_MAX_SEGMENTS, DsaPointerIsValid, get_segment_by_index(), dsa_segment_map::mapped_address, dsa_area::segment_maps, and unlikely.

Referenced by add_span_to_fullness_class(), alloc_object(), delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_dump(), dsa_free(), dsa_trim(), dshash_attach(), dshash_create(), dshash_destroy(), dshash_dump(), ensure_active_superblock(), ensure_valid_bucket_pointers(), ExecHashTableDetachBatch(), ExecParallelHashEnsureBatchAccessors(), ExecParallelHashFirstTuple(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashJoinSetUpBatches(), ExecParallelHashNextTuple(), ExecParallelHashPopChunkQueue(), ExecParallelHashRepartitionRest(), ExecParallelHashTableAlloc(), ExecParallelHashTableSetCurrentBatch(), ExecParallelHashTupleAlloc(), find_in_bucket(), find_or_make_matching_shared_tupledesc(), init_span(), insert_into_bucket(), insert_item_into_bucket(), lookup_rowtype_tupdesc_internal(), pagetable_allocate(), ParallelQueryMain(), resize(), SerializeParamExecParams(), share_tupledesc(), shared_record_table_compare(), shared_record_table_hash(), tbm_attach_shared_iterate(), tbm_free_shared_area(), tbm_prepare_shared_iterate(), transfer_first_span(), and unlink_span().

933 {
935  size_t offset;
936 
937  /* Convert InvalidDsaPointer to NULL. */
938  if (!DsaPointerIsValid(dp))
939  return NULL;
940 
941  /* Process any requests to detach from freed segments. */
943 
944  /* Break the dsa_pointer into its components. */
945  index = DSA_EXTRACT_SEGMENT_NUMBER(dp);
946  offset = DSA_EXTRACT_OFFSET(dp);
947  Assert(index < DSA_MAX_SEGMENTS);
948 
949  /* Check if we need to cause this segment to be mapped in. */
950  if (unlikely(area->segment_maps[index].mapped_address == NULL))
951  {
952  /* Call for effect (we don't need the result). */
953  get_segment_by_index(area, index);
954  }
955 
956  return area->segment_maps[index].mapped_address + offset;
957 }
Definition: type.h:89
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
Definition: dsa.c:118
#define Assert(condition)
Definition: c.h:804
#define DSA_EXTRACT_OFFSET(dp)
Definition: dsa.c:121
size_t dsa_segment_index
Definition: dsa.c:124
char * mapped_address
Definition: dsa.c:342
#define DSA_MAX_SEGMENTS
Definition: dsa.c:94
#define DsaPointerIsValid(x)
Definition: dsa.h:81
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
static void check_for_freed_segments(dsa_area *area)
Definition: dsa.c:2229
#define unlikely(x)
Definition: c.h:273
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ dsa_get_handle()

dsa_handle dsa_get_handle ( dsa_area area)

Definition at line 494 of file dsa.c.

References Assert, dsa_area::control, DSM_HANDLE_INVALID, and dsa_area_control::handle.

495 {
497  return area->control->handle;
498 }
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
dsa_area_control * control
Definition: dsa.c:357
#define Assert(condition)
Definition: c.h:804
dsa_handle handle
Definition: dsa.c:303

◆ dsa_minimum_size()

size_t dsa_minimum_size ( void  )

Definition at line 1169 of file dsa.c.

References FPM_PAGE_SIZE, and MAXALIGN.

Referenced by create_internal(), and ExecInitParallelPlan().

1170 {
1171  size_t size;
1172  int pages = 0;
1173 
1174  size = MAXALIGN(sizeof(dsa_area_control)) +
1175  MAXALIGN(sizeof(FreePageManager));
1176 
1177  /* Figure out how many pages we need, including the page map... */
1178  while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
1179  {
1180  ++pages;
1181  size += sizeof(dsa_pointer);
1182  }
1183 
1184  return pages * FPM_PAGE_SIZE;
1185 }
uint64 dsa_pointer
Definition: dsa.h:62
#define FPM_PAGE_SIZE
Definition: freepage.h:30
#define MAXALIGN(LEN)
Definition: c.h:757

◆ dsa_on_dsm_detach_release_in_place()

void dsa_on_dsm_detach_release_in_place ( dsm_segment segment,
Datum  place 
)

Definition at line 572 of file dsa.c.

References DatumGetPointer, and dsa_release_in_place().

Referenced by dsa_attach(), dsa_attach_in_place(), dsa_create(), and dsa_create_in_place().

573 {
575 }
void dsa_release_in_place(void *place)
Definition: dsa.c:601
#define DatumGetPointer(X)
Definition: postgres.h:593

◆ dsa_on_shmem_exit_release_in_place()

void dsa_on_shmem_exit_release_in_place ( int  code,
Datum  place 
)

Definition at line 586 of file dsa.c.

References DatumGetPointer, and dsa_release_in_place().

587 {
589 }
void dsa_release_in_place(void *place)
Definition: dsa.c:601
#define DatumGetPointer(X)
Definition: postgres.h:593

◆ dsa_pin()

void dsa_pin ( dsa_area area)

Definition at line 965 of file dsa.c.

References dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

966 {
968  if (area->control->pinned)
969  {
971  elog(ERROR, "dsa_area already pinned");
972  }
973  area->control->pinned = true;
974  ++area->control->refcnt;
976 }
bool pinned
Definition: dsa.c:319
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
int refcnt
Definition: dsa.c:317
#define ERROR
Definition: elog.h:46
dsa_area_control * control
Definition: dsa.c:357
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
#define elog(elevel,...)
Definition: elog.h:232
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143

◆ dsa_pin_mapping()

void dsa_pin_mapping ( dsa_area area)

Definition at line 631 of file dsa.c.

References Assert, dsm_pin_mapping(), dsa_area::high_segment_index, i, dsa_area::mapping_pinned, dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AttachSession(), and GetSessionDsmHandle().

632 {
633  int i;
634 
635  Assert(!area->mapping_pinned);
636  area->mapping_pinned = true;
637 
638  for (i = 0; i <= area->high_segment_index; ++i)
639  if (area->segment_maps[i].segment != NULL)
641 }
bool mapping_pinned
Definition: dsa.c:360
dsa_segment_index high_segment_index
Definition: dsa.c:371
dsm_segment * segment
Definition: dsa.c:341
void dsm_pin_mapping(dsm_segment *seg)
Definition: dsm.c:881
#define Assert(condition)
Definition: c.h:804
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
int i

◆ dsa_release_in_place()

void dsa_release_in_place ( void *  place)

Definition at line 601 of file dsa.c.

References Assert, dsa_area::control, DSA_SEGMENT_HEADER_MAGIC, DSM_HANDLE_INVALID, dsm_unpin_segment(), dsa_area_control::handle, dsa_area_control::high_segment_index, i, dsa_area_control::lock, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_area_control::refcnt, dsa_area_control::segment_handles, and dsa_area_control::segment_header.

Referenced by dsa_on_dsm_detach_release_in_place(), and dsa_on_shmem_exit_release_in_place().

602 {
603  dsa_area_control *control = (dsa_area_control *) place;
604  int i;
605 
606  LWLockAcquire(&control->lock, LW_EXCLUSIVE);
607  Assert(control->segment_header.magic ==
608  (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
609  Assert(control->refcnt > 0);
610  if (--control->refcnt == 0)
611  {
612  for (i = 0; i <= control->high_segment_index; ++i)
613  {
614  dsm_handle handle;
615 
616  handle = control->segment_handles[i];
617  if (handle != DSM_HANDLE_INVALID)
618  dsm_unpin_segment(handle);
619  }
620  }
621  LWLockRelease(&control->lock);
622 }
uint32 dsm_handle
Definition: dsm_impl.h:55
dsa_segment_index high_segment_index
Definition: dsa.c:315
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
LWLock lock
Definition: dsa.c:325
int refcnt
Definition: dsa.c:317
void dsm_unpin_segment(dsm_handle handle)
Definition: dsm.c:953
dsa_segment_header segment_header
Definition: dsa.c:301
#define Assert(condition)
Definition: c.h:804
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:305
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
int i
#define DSA_SEGMENT_HEADER_MAGIC
Definition: dsa.c:111
uint32 magic
Definition: dsa.c:154
dsa_handle handle
Definition: dsa.c:303

◆ dsa_set_size_limit()

void dsa_set_size_limit ( dsa_area area,
size_t  limit 
)

Definition at line 1008 of file dsa.c.

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), and dsa_area_control::max_total_segment_size.

1009 {
1011  area->control->max_total_segment_size = limit;
1013 }
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
dsa_area_control * control
Definition: dsa.c:357
size_t max_total_segment_size
Definition: dsa.c:313
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143

◆ dsa_trim()

void dsa_trim ( dsa_area area)

Definition at line 1020 of file dsa.c.

References dsa_area::control, destroy_superblock(), dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DsaPointerIsValid, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_control::pools, and dsa_area_pool::spans.

1021 {
1022  int size_class;
1023 
1024  /*
1025  * Trim in reverse pool order so we get to the spans-of-spans last, just
1026  * in case any become entirely free while processing all the other pools.
1027  */
1028  for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
1029  {
1030  dsa_area_pool *pool = &area->control->pools[size_class];
1031  dsa_pointer span_pointer;
1032 
1033  if (size_class == DSA_SCLASS_SPAN_LARGE)
1034  {
1035  /* Large object frees give back segments aggressively already. */
1036  continue;
1037  }
1038 
1039  /*
1040  * Search fullness class 1 only. That is where we expect to find an
1041  * entirely empty superblock (entirely empty superblocks in other
1042  * fullness classes are returned to the free page map by dsa_free).
1043  */
1044  LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
1045  span_pointer = pool->spans[1];
1046  while (DsaPointerIsValid(span_pointer))
1047  {
1048  dsa_area_span *span = dsa_get_address(area, span_pointer);
1049  dsa_pointer next = span->nextspan;
1050 
1051  if (span->nallocatable == span->nmax)
1052  destroy_superblock(area, span_pointer);
1053 
1054  span_pointer = next;
1055  }
1056  LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
1057  }
1058 }
uint16 nmax
Definition: dsa.c:205
dsa_pointer nextspan
Definition: dsa.c:198
static int32 next
Definition: blutils.c:219
#define DSA_NUM_SIZE_CLASSES
Definition: dsa.c:247
uint64 dsa_pointer
Definition: dsa.h:62
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
dsa_area_control * control
Definition: dsa.c:357
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
Definition: dsa.c:309
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
Definition: dsa.c:1800
uint16 nallocatable
Definition: dsa.c:203
#define DSA_SCLASS_SPAN_LARGE
Definition: dsa.c:251

◆ dsa_unpin()

void dsa_unpin ( dsa_area area)

Definition at line 984 of file dsa.c.

References Assert, dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

985 {
987  Assert(area->control->refcnt > 1);
988  if (!area->control->pinned)
989  {
991  elog(ERROR, "dsa_area not pinned");
992  }
993  area->control->pinned = false;
994  --area->control->refcnt;
996 }
bool pinned
Definition: dsa.c:319
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
int refcnt
Definition: dsa.c:317
#define ERROR
Definition: elog.h:46
dsa_area_control * control
Definition: dsa.c:357
#define Assert(condition)
Definition: c.h:804
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
#define elog(elevel,...)
Definition: elog.h:232
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143

◆ ensure_active_superblock()

static bool ensure_active_superblock ( dsa_area area,
dsa_area_pool pool,
int  size_class 
)
static

Definition at line 1525 of file dsa.c.

References alloc_object(), Assert, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_MAKE_POINTER, DSA_PAGES_PER_SUPERBLOCK, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SUPERBLOCK_SIZE, DsaPointerIsValid, elog, FATAL, dsa_area_span::fclass, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerGet(), get_best_segment(), get_segment_index, i, init_span(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), make_new_segment(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_segment_map::pagemap, dsa_area_span::prevspan, dsa_area_pool::spans, and transfer_first_span().

Referenced by alloc_object().

1527 {
1528  dsa_pointer span_pointer;
1529  dsa_pointer start_pointer;
1530  size_t obsize = dsa_size_classes[size_class];
1531  size_t nmax;
1532  int fclass;
1533  size_t npages = 1;
1534  size_t first_page;
1535  size_t i;
1536  dsa_segment_map *segment_map;
1537 
1538  Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1539 
1540  /*
1541  * Compute the number of objects that will fit in a block of this size
1542  * class. Span-of-spans blocks are just a single page, and the first
1543  * object isn't available for use because it describes the block-of-spans
1544  * itself.
1545  */
1546  if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1547  nmax = FPM_PAGE_SIZE / obsize - 1;
1548  else
1549  nmax = DSA_SUPERBLOCK_SIZE / obsize;
1550 
1551  /*
1552  * If fullness class 1 is empty, try to find a span to put in it by
1553  * scanning higher-numbered fullness classes (excluding the last one,
1554  * whose blocks are certain to all be completely full).
1555  */
1556  for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
1557  {
1558  span_pointer = pool->spans[fclass];
1559 
1560  while (DsaPointerIsValid(span_pointer))
1561  {
1562  int tfclass;
1563  dsa_area_span *span;
1564  dsa_area_span *nextspan;
1565  dsa_area_span *prevspan;
1566  dsa_pointer next_span_pointer;
1567 
1568  span = (dsa_area_span *)
1569  dsa_get_address(area, span_pointer);
1570  next_span_pointer = span->nextspan;
1571 
1572  /* Figure out what fullness class should contain this span. */
1573  tfclass = (nmax - span->nallocatable)
1574  * (DSA_FULLNESS_CLASSES - 1) / nmax;
1575 
1576  /* Look up next span. */
1577  if (DsaPointerIsValid(span->nextspan))
1578  nextspan = (dsa_area_span *)
1579  dsa_get_address(area, span->nextspan);
1580  else
1581  nextspan = NULL;
1582 
1583  /*
1584  * If utilization has dropped enough that this now belongs in some
1585  * other fullness class, move it there.
1586  */
1587  if (tfclass < fclass)
1588  {
1589  /* Remove from the current fullness class list. */
1590  if (pool->spans[fclass] == span_pointer)
1591  {
1592  /* It was the head; remove it. */
1594  pool->spans[fclass] = span->nextspan;
1595  if (nextspan != NULL)
1596  nextspan->prevspan = InvalidDsaPointer;
1597  }
1598  else
1599  {
1600  /* It was not the head. */
1602  prevspan = (dsa_area_span *)
1603  dsa_get_address(area, span->prevspan);
1604  prevspan->nextspan = span->nextspan;
1605  }
1606  if (nextspan != NULL)
1607  nextspan->prevspan = span->prevspan;
1608 
1609  /* Push onto the head of the new fullness class list. */
1610  span->nextspan = pool->spans[tfclass];
1611  pool->spans[tfclass] = span_pointer;
1612  span->prevspan = InvalidDsaPointer;
1613  if (DsaPointerIsValid(span->nextspan))
1614  {
1615  nextspan = (dsa_area_span *)
1616  dsa_get_address(area, span->nextspan);
1617  nextspan->prevspan = span_pointer;
1618  }
1619  span->fclass = tfclass;
1620  }
1621 
1622  /* Advance to next span on list. */
1623  span_pointer = next_span_pointer;
1624  }
1625 
1626  /* Stop now if we found a suitable block. */
1627  if (DsaPointerIsValid(pool->spans[1]))
1628  return true;
1629  }
1630 
1631  /*
1632  * If there are no blocks that properly belong in fullness class 1, pick
1633  * one from some other fullness class and move it there anyway, so that we
1634  * have an allocation target. Our last choice is to transfer a block
1635  * that's almost empty (and might become completely empty soon if left
1636  * alone), but even that is better than failing, which is what we must do
1637  * if there are no blocks at all with freespace.
1638  */
1639  Assert(!DsaPointerIsValid(pool->spans[1]));
1640  for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
1641  if (transfer_first_span(area, pool, fclass, 1))
1642  return true;
1643  if (!DsaPointerIsValid(pool->spans[1]) &&
1644  transfer_first_span(area, pool, 0, 1))
1645  return true;
1646 
1647  /*
1648  * We failed to find an existing span with free objects, so we need to
1649  * allocate a new superblock and construct a new span to manage it.
1650  *
1651  * First, get a dsa_area_span object to describe the new superblock block
1652  * ... unless this allocation is for a dsa_area_span object, in which case
1653  * that's surely not going to work. We handle that case by storing the
1654  * span describing a block-of-spans inline.
1655  */
1656  if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
1657  {
1658  span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
1659  if (!DsaPointerIsValid(span_pointer))
1660  return false;
1661  npages = DSA_PAGES_PER_SUPERBLOCK;
1662  }
1663 
1664  /* Find or create a segment and allocate the superblock. */
1666  segment_map = get_best_segment(area, npages);
1667  if (segment_map == NULL)
1668  {
1669  segment_map = make_new_segment(area, npages);
1670  if (segment_map == NULL)
1671  {
1673  return false;
1674  }
1675  }
1676 
1677  /*
1678  * This shouldn't happen: get_best_segment() or make_new_segment()
1679  * promised that we can successfully allocate npages.
1680  */
1681  if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
1682  elog(FATAL,
1683  "dsa_allocate could not find %zu free pages for superblock",
1684  npages);
1686 
1687  /* Compute the start of the superblock. */
1688  start_pointer =
1689  DSA_MAKE_POINTER(get_segment_index(area, segment_map),
1690  first_page * FPM_PAGE_SIZE);
1691 
1692  /*
1693  * If this is a block-of-spans, carve the descriptor right out of the
1694  * allocated space.
1695  */
1696  if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1697  {
1698  /*
1699  * We have a pointer into the segment. We need to build a dsa_pointer
1700  * from the segment index and offset into the segment.
1701  */
1702  span_pointer = start_pointer;
1703  }
1704 
1705  /* Initialize span and pagemap. */
1706  init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1707  for (i = 0; i < npages; ++i)
1708  segment_map->pagemap[first_page + i] = span_pointer;
1709 
1710  return true;
1711 }
#define DSA_SUPERBLOCK_SIZE
Definition: dsa.c:378
dsa_pointer nextspan
Definition: dsa.c:198
#define InvalidDsaPointer
Definition: dsa.h:78
bool LWLockHeldByMe(LWLock *l)
Definition: lwlock.c:1932
#define DSA_FULLNESS_CLASSES
Definition: dsa.c:277
uint64 dsa_pointer
Definition: dsa.h:62
FreePageManager * fpm
Definition: dsa.c:344
uint16 fclass
Definition: dsa.c:206
#define get_segment_index(area, segment_map_ptr)
Definition: dsa.c:381
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1816
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
Definition: freepage.c:210
#define FATAL
Definition: elog.h:49
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
Definition: dsa.c:1397
static const uint16 dsa_size_classes[]
Definition: dsa.c:236
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
dsa_pointer prevspan
Definition: dsa.c:197
#define FPM_PAGE_SIZE
Definition: freepage.h:30
#define DSA_SCLASS_BLOCK_OF_SPANS
Definition: dsa.c:250
#define Assert(condition)
Definition: c.h:804
#define DSA_MAKE_POINTER(segment_number, offset)
Definition: dsa.c:114
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1203
dsa_pointer * pagemap
Definition: dsa.c:345
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81
uint16 nallocatable
Definition: dsa.c:203
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
Definition: dsa.c:2060
#define elog(elevel,...)
Definition: elog.h:232
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
Definition: dsa.c:1342
int i
static dsa_pointer alloc_object(dsa_area *area, int size_class)
Definition: dsa.c:1437
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
#define DSA_PAGES_PER_SUPERBLOCK
Definition: dsa.c:104
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
Definition: dsa.c:1968

◆ get_best_segment()

static dsa_segment_map * get_best_segment ( dsa_area area,
size_t  npages 
)
static

Definition at line 1968 of file dsa.c.

References Assert, dsa_segment_header::bin, check_for_freed_segments_locked(), contiguous_pages_to_segment_bin, dsa_area::control, DSA_AREA_LOCK, DSA_NUM_SEGMENT_BINS, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), dsa_segment_map::header, LWLockHeldByMe(), dsa_segment_header::next, next, dsa_segment_header::prev, dsa_area_control::segment_bins, and unlink_segment().

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

1969 {
1970  size_t bin;
1971 
1974 
1975  /*
1976  * Start searching from the first bin that *might* have enough contiguous
1977  * pages.
1978  */
1979  for (bin = contiguous_pages_to_segment_bin(npages);
1980  bin < DSA_NUM_SEGMENT_BINS;
1981  ++bin)
1982  {
1983  /*
1984  * The minimum contiguous size that any segment in this bin should
1985  * have. We'll re-bin if we see segments with fewer.
1986  */
1987  size_t threshold = (size_t) 1 << (bin - 1);
1988  dsa_segment_index segment_index;
1989 
1990  /* Search this bin for a segment with enough contiguous space. */
1991  segment_index = area->control->segment_bins[bin];
1992  while (segment_index != DSA_SEGMENT_INDEX_NONE)
1993  {
1994  dsa_segment_map *segment_map;
1995  dsa_segment_index next_segment_index;
1996  size_t contiguous_pages;
1997 
1998  segment_map = get_segment_by_index(area, segment_index);
1999  next_segment_index = segment_map->header->next;
2000  contiguous_pages = fpm_largest(segment_map->fpm);
2001 
2002  /* Not enough for the request, still enough for this bin. */
2003  if (contiguous_pages >= threshold && contiguous_pages < npages)
2004  {
2005  segment_index = next_segment_index;
2006  continue;
2007  }
2008 
2009  /* Re-bin it if it's no longer in the appropriate bin. */
2010  if (contiguous_pages < threshold)
2011  {
2012  size_t new_bin;
2013 
2014  new_bin = contiguous_pages_to_segment_bin(contiguous_pages);
2015 
2016  /* Remove it from its current bin. */
2017  unlink_segment(area, segment_map);
2018 
2019  /* Push it onto the front of its new bin. */
2020  segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
2021  segment_map->header->next =
2022  area->control->segment_bins[new_bin];
2023  segment_map->header->bin = new_bin;
2024  area->control->segment_bins[new_bin] = segment_index;
2025  if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2026  {
2028 
2029  next = get_segment_by_index(area,
2030  segment_map->header->next);
2031  Assert(next->header->bin == new_bin);
2032  next->header->prev = segment_index;
2033  }
2034 
2035  /*
2036  * But fall through to see if it's enough to satisfy this
2037  * request anyway....
2038  */
2039  }
2040 
2041  /* Check if we are done. */
2042  if (contiguous_pages >= npages)
2043  return segment_map;
2044 
2045  /* Continue searching the same bin. */
2046  segment_index = next_segment_index;
2047  }
2048  }
2049 
2050  /* Not found. */
2051  return NULL;
2052 }
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
Definition: dsa.c:307
static int32 next
Definition: blutils.c:219
#define contiguous_pages_to_segment_bin(n)
Definition: dsa.c:140
bool LWLockHeldByMe(LWLock *l)
Definition: lwlock.c:1932
dsa_segment_index prev
Definition: dsa.c:164
FreePageManager * fpm
Definition: dsa.c:344
dsa_segment_header * header
Definition: dsa.c:343
dsa_segment_index next
Definition: dsa.c:170
dsa_area_control * control
Definition: dsa.c:357
#define DSA_NUM_SEGMENT_BINS
Definition: dsa.c:133
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
Definition: dsa.c:1936
#define Assert(condition)
Definition: c.h:804
#define fpm_largest(fpm)
Definition: freepage.h:88
#define DSA_SEGMENT_INDEX_NONE
Definition: dsa.c:127
size_t dsa_segment_index
Definition: dsa.c:124
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
size_t bin
Definition: dsa.c:172
static void check_for_freed_segments_locked(dsa_area *area)
Definition: dsa.c:2265
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ get_segment_by_index()

static dsa_segment_map * get_segment_by_index ( dsa_area area,
dsa_segment_index  index 
)
static

Definition at line 1722 of file dsa.c.

References Assert, dsa_area::control, DSA_SEGMENT_HEADER_MAGIC, dsm_attach(), DSM_HANDLE_INVALID, dsm_pin_mapping(), dsm_segment_address(), elog, ERROR, dsa_segment_map::fpm, dsa_segment_header::freed, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area::mapping_pinned, MAXALIGN, dsa_segment_map::pagemap, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area::segment_maps, and unlikely.

Referenced by destroy_superblock(), dsa_dump(), dsa_free(), dsa_get_address(), get_best_segment(), make_new_segment(), and unlink_segment().

1723 {
1724  if (unlikely(area->segment_maps[index].mapped_address == NULL))
1725  {
1726  dsm_handle handle;
1727  dsm_segment *segment;
1728  dsa_segment_map *segment_map;
1729 
1730  /*
1731  * If we are reached by dsa_free or dsa_get_address, there must be at
1732  * least one object allocated in the referenced segment. Otherwise,
1733  * their caller has a double-free or access-after-free bug, which we
1734  * have no hope of detecting. So we know it's safe to access this
1735  * array slot without holding a lock; it won't change underneath us.
1736  * Furthermore, we know that we can see the latest contents of the
1737  * slot, as explained in check_for_freed_segments, which those
1738  * functions call before arriving here.
1739  */
1740  handle = area->control->segment_handles[index];
1741 
1742  /* It's an error to try to access an unused slot. */
1743  if (handle == DSM_HANDLE_INVALID)
1744  elog(ERROR,
1745  "dsa_area could not attach to a segment that has been freed");
1746 
1747  segment = dsm_attach(handle);
1748  if (segment == NULL)
1749  elog(ERROR, "dsa_area could not attach to segment");
1750  if (area->mapping_pinned)
1751  dsm_pin_mapping(segment);
1752  segment_map = &area->segment_maps[index];
1753  segment_map->segment = segment;
1754  segment_map->mapped_address = dsm_segment_address(segment);
1755  segment_map->header =
1756  (dsa_segment_header *) segment_map->mapped_address;
1757  segment_map->fpm = (FreePageManager *)
1758  (segment_map->mapped_address +
1759  MAXALIGN(sizeof(dsa_segment_header)));
1760  segment_map->pagemap = (dsa_pointer *)
1761  (segment_map->mapped_address +
1762  MAXALIGN(sizeof(dsa_segment_header)) +
1763  MAXALIGN(sizeof(FreePageManager)));
1764 
1765  /* Remember the highest index this backend has ever mapped. */
1766  if (area->high_segment_index < index)
1767  area->high_segment_index = index;
1768 
1769  Assert(segment_map->header->magic ==
1771  }
1772 
1773  /*
1774  * Callers of dsa_get_address() and dsa_free() don't hold the area lock,
1775  * but it's a bug in the calling code and undefined behavior if the
1776  * address is not live (ie if the segment might possibly have been freed,
1777  * they're trying to use a dangling pointer).
1778  *
1779  * For dsa.c code that holds the area lock to manipulate segment_bins
1780  * lists, it would be a bug if we ever reach a freed segment here. After
1781  * it's marked as freed, the only thing any backend should do with it is
1782  * unmap it, and it should always have done that in
1783  * check_for_freed_segments_locked() before arriving here to resolve an
1784  * index to a segment_map.
1785  *
1786  * Either way we can assert that we aren't returning a freed segment.
1787  */
1788  Assert(!area->segment_maps[index].header->freed);
1789 
1790  return &area->segment_maps[index];
1791 }
bool mapping_pinned
Definition: dsa.c:360
uint32 dsm_handle
Definition: dsm_impl.h:55
dsm_segment * dsm_attach(dsm_handle h)
Definition: dsm.c:631
dsa_segment_index high_segment_index
Definition: dsa.c:371
dsm_segment * segment
Definition: dsa.c:341
uint64 dsa_pointer
Definition: dsa.h:62
FreePageManager * fpm
Definition: dsa.c:344
dsa_segment_header * header
Definition: dsa.c:343
Definition: type.h:89
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
#define ERROR
Definition: elog.h:46
void dsm_pin_mapping(dsm_segment *seg)
Definition: dsm.c:881
dsa_area_control * control
Definition: dsa.c:357
void * dsm_segment_address(dsm_segment *seg)
Definition: dsm.c:1059
#define Assert(condition)
Definition: c.h:804
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:305
#define MAXALIGN(LEN)
Definition: c.h:757
dsa_pointer * pagemap
Definition: dsa.c:345
char * mapped_address
Definition: dsa.c:342
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
#define elog(elevel,...)
Definition: elog.h:232
#define unlikely(x)
Definition: c.h:273
#define DSA_SEGMENT_HEADER_MAGIC
Definition: dsa.c:111
uint32 magic
Definition: dsa.c:154
dsa_handle handle
Definition: dsa.c:303

◆ init_span()

static void init_span ( dsa_area area,
dsa_pointer  span_pointer,
dsa_area_pool pool,
dsa_pointer  start,
size_t  npages,
uint16  size_class 
)
static

Definition at line 1342 of file dsa.c.

References Assert, dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DSA_SUPERBLOCK_SIZE, DsaAreaPoolToDsaPointer, DsaPointerIsValid, dsa_area_span::fclass, dsa_area_span::firstfree, FPM_PAGE_SIZE, InvalidDsaPointer, LWLockHeldByMe(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_area_span::ninitialized, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_span::pool, dsa_area_span::prevspan, dsa_area_span::size_class, dsa_area_pool::spans, and dsa_area_span::start.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

1346 {
1347  dsa_area_span *span = dsa_get_address(area, span_pointer);
1348  size_t obsize = dsa_size_classes[size_class];
1349 
1350  /*
1351  * The per-pool lock must be held because we manipulate the span list for
1352  * this pool.
1353  */
1354  Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1355 
1356  /* Push this span onto the front of the span list for fullness class 1. */
1357  if (DsaPointerIsValid(pool->spans[1]))
1358  {
1359  dsa_area_span *head = (dsa_area_span *)
1360  dsa_get_address(area, pool->spans[1]);
1361 
1362  head->prevspan = span_pointer;
1363  }
1364  span->pool = DsaAreaPoolToDsaPointer(area, pool);
1365  span->nextspan = pool->spans[1];
1366  span->prevspan = InvalidDsaPointer;
1367  pool->spans[1] = span_pointer;
1368 
1369  span->start = start;
1370  span->npages = npages;
1371  span->size_class = size_class;
1372  span->ninitialized = 0;
1373  if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1374  {
1375  /*
1376  * A block-of-spans contains its own descriptor, so mark one object as
1377  * initialized and reduce the count of allocatable objects by one.
1378  * Doing this here has the side effect of also reducing nmax by one,
1379  * which is important to make sure we free this object at the correct
1380  * time.
1381  */
1382  span->ninitialized = 1;
1383  span->nallocatable = FPM_PAGE_SIZE / obsize - 1;
1384  }
1385  else if (size_class != DSA_SCLASS_SPAN_LARGE)
1386  span->nallocatable = DSA_SUPERBLOCK_SIZE / obsize;
1388  span->nmax = span->nallocatable;
1389  span->fclass = 1;
1390 }
uint16 nmax
Definition: dsa.c:205
#define DSA_SPAN_NOTHING_FREE
Definition: dsa.c:377
#define DSA_SUPERBLOCK_SIZE
Definition: dsa.c:378
dsa_pointer nextspan
Definition: dsa.c:198
#define InvalidDsaPointer
Definition: dsa.h:78
uint16 ninitialized
Definition: dsa.c:202
bool LWLockHeldByMe(LWLock *l)
Definition: lwlock.c:1932
dsa_pointer start
Definition: dsa.c:199
size_t npages
Definition: dsa.c:200
uint16 fclass
Definition: dsa.c:206
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
uint16 size_class
Definition: dsa.c:201
static const uint16 dsa_size_classes[]
Definition: dsa.c:236
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:144
dsa_pointer prevspan
Definition: dsa.c:197
#define FPM_PAGE_SIZE
Definition: freepage.h:30
#define DSA_SCLASS_BLOCK_OF_SPANS
Definition: dsa.c:250
#define Assert(condition)
Definition: c.h:804
dsa_pointer pool
Definition: dsa.c:196
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81
uint16 nallocatable
Definition: dsa.c:203
uint16 firstfree
Definition: dsa.c:204
#define DSA_SCLASS_SPAN_LARGE
Definition: dsa.c:251
#define DsaAreaPoolToDsaPointer(area, p)
Definition: dsa.c:329

◆ make_new_segment()

static dsa_segment_map * make_new_segment ( dsa_area area,
size_t  requested_pages 
)
static

Definition at line 2060 of file dsa.c.

References Assert, dsa_segment_header::bin, contiguous_pages_to_segment_bin, dsa_area::control, DSA_AREA_LOCK, DSA_INITIAL_SEGMENT_SIZE, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_NUM_SEGMENTS_AT_EACH_SIZE, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, dsm_create(), DSM_HANDLE_INVALID, dsm_pin_mapping(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, FreePageManagerInitialize(), FreePageManagerPut(), get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, dsa_area_control::high_segment_index, dsa_area::high_segment_index, LWLockHeldByMe(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area::mapping_pinned, dsa_area_control::max_total_segment_size, MAXALIGN, Min, dsa_segment_header::next, next, dsa_segment_map::pagemap, dsa_segment_header::prev, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, total_size, and dsa_segment_header::usable_pages.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

2061 {
2062  dsa_segment_index new_index;
2063  size_t metadata_bytes;
2064  size_t total_size;
2065  size_t total_pages;
2066  size_t usable_pages;
2067  dsa_segment_map *segment_map;
2068  dsm_segment *segment;
2069 
2071 
2072  /* Find a segment slot that is not in use (linearly for now). */
2073  for (new_index = 1; new_index < DSA_MAX_SEGMENTS; ++new_index)
2074  {
2075  if (area->control->segment_handles[new_index] == DSM_HANDLE_INVALID)
2076  break;
2077  }
2078  if (new_index == DSA_MAX_SEGMENTS)
2079  return NULL;
2080 
2081  /*
2082  * If the total size limit is already exceeded, then we exit early and
2083  * avoid arithmetic wraparound in the unsigned expressions below.
2084  */
2085  if (area->control->total_segment_size >=
2087  return NULL;
2088 
2089  /*
2090  * The size should be at least as big as requested, and at least big
2091  * enough to follow a geometric series that approximately doubles the
2092  * total storage each time we create a new segment. We use geometric
2093  * growth because the underlying DSM system isn't designed for large
2094  * numbers of segments (otherwise we might even consider just using one
2095  * DSM segment for each large allocation and for each superblock, and then
2096  * we wouldn't need to use FreePageManager).
2097  *
2098  * We decide on a total segment size first, so that we produce tidy
2099  * power-of-two sized segments. This is a good property to have if we
2100  * move to huge pages in the future. Then we work back to the number of
2101  * pages we can fit.
2102  */
2103  total_size = DSA_INITIAL_SEGMENT_SIZE *
2104  ((size_t) 1 << (new_index / DSA_NUM_SEGMENTS_AT_EACH_SIZE));
2105  total_size = Min(total_size, DSA_MAX_SEGMENT_SIZE);
2106  total_size = Min(total_size,
2108  area->control->total_segment_size);
2109 
2110  total_pages = total_size / FPM_PAGE_SIZE;
2111  metadata_bytes =
2112  MAXALIGN(sizeof(dsa_segment_header)) +
2113  MAXALIGN(sizeof(FreePageManager)) +
2114  sizeof(dsa_pointer) * total_pages;
2115 
2116  /* Add padding up to next page boundary. */
2117  if (metadata_bytes % FPM_PAGE_SIZE != 0)
2118  metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
2119  if (total_size <= metadata_bytes)
2120  return NULL;
2121  usable_pages = (total_size - metadata_bytes) / FPM_PAGE_SIZE;
2122  Assert(metadata_bytes + usable_pages * FPM_PAGE_SIZE <= total_size);
2123 
2124  /* See if that is enough... */
2125  if (requested_pages > usable_pages)
2126  {
2127  /*
2128  * We'll make an odd-sized segment, working forward from the requested
2129  * number of pages.
2130  */
2131  usable_pages = requested_pages;
2132  metadata_bytes =
2133  MAXALIGN(sizeof(dsa_segment_header)) +
2134  MAXALIGN(sizeof(FreePageManager)) +
2135  usable_pages * sizeof(dsa_pointer);
2136 
2137  /* Add padding up to next page boundary. */
2138  if (metadata_bytes % FPM_PAGE_SIZE != 0)
2139  metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
2140  total_size = metadata_bytes + usable_pages * FPM_PAGE_SIZE;
2141 
2142  /* Is that too large for dsa_pointer's addressing scheme? */
2143  if (total_size > DSA_MAX_SEGMENT_SIZE)
2144  return NULL;
2145 
2146  /* Would that exceed the limit? */
2147  if (total_size > area->control->max_total_segment_size -
2148  area->control->total_segment_size)
2149  return NULL;
2150  }
2151 
2152  /* Create the segment. */
2153  segment = dsm_create(total_size, 0);
2154  if (segment == NULL)
2155  return NULL;
2156  dsm_pin_segment(segment);
2157  if (area->mapping_pinned)
2158  dsm_pin_mapping(segment);
2159 
2160  /* Store the handle in shared memory to be found by index. */
2161  area->control->segment_handles[new_index] =
2162  dsm_segment_handle(segment);
2163  /* Track the highest segment index in the history of the area. */
2164  if (area->control->high_segment_index < new_index)
2165  area->control->high_segment_index = new_index;
2166  /* Track the highest segment index this backend has ever mapped. */
2167  if (area->high_segment_index < new_index)
2168  area->high_segment_index = new_index;
2169  /* Track total size of all segments. */
2173 
2174  /* Build a segment map for this segment in this backend. */
2175  segment_map = &area->segment_maps[new_index];
2176  segment_map->segment = segment;
2177  segment_map->mapped_address = dsm_segment_address(segment);
2178  segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
2179  segment_map->fpm = (FreePageManager *)
2180  (segment_map->mapped_address +
2181  MAXALIGN(sizeof(dsa_segment_header)));
2182  segment_map->pagemap = (dsa_pointer *)
2183  (segment_map->mapped_address +
2184  MAXALIGN(sizeof(dsa_segment_header)) +
2185  MAXALIGN(sizeof(FreePageManager)));
2186 
2187  /* Set up the free page map. */
2188  FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
2189  FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
2190  usable_pages);
2191 
2192  /* Set up the segment header and put it in the appropriate bin. */
2193  segment_map->header->magic =
2194  DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ new_index;
2195  segment_map->header->usable_pages = usable_pages;
2196  segment_map->header->size = total_size;
2197  segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
2198  segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
2199  segment_map->header->next =
2200  area->control->segment_bins[segment_map->header->bin];
2201  segment_map->header->freed = false;
2202  area->control->segment_bins[segment_map->header->bin] = new_index;
2203  if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2204  {
2206  get_segment_by_index(area, segment_map->header->next);
2207 
2208  Assert(next->header->bin == segment_map->header->bin);
2209  next->header->prev = new_index;
2210  }
2211 
2212  return segment_map;
2213 }
bool mapping_pinned
Definition: dsa.c:360
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
Definition: dsa.c:307
static int32 next
Definition: blutils.c:219
#define contiguous_pages_to_segment_bin(n)
Definition: dsa.c:140
bool LWLockHeldByMe(LWLock *l)
Definition: lwlock.c:1932
size_t usable_pages
Definition: dsa.c:156
dsa_segment_index high_segment_index
Definition: dsa.c:371
#define Min(x, y)
Definition: c.h:986
dsa_segment_index high_segment_index
Definition: dsa.c:315
dsm_segment * segment
Definition: dsa.c:341
size_t size
Definition: dsa.c:158
dsa_segment_index prev
Definition: dsa.c:164
dsm_handle dsm_segment_handle(dsm_segment *seg)
Definition: dsm.c:1087
size_t total_segment_size
Definition: dsa.c:311
uint64 dsa_pointer
Definition: dsa.h:62
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
Definition: freepage.c:379
FreePageManager * fpm
Definition: dsa.c:344
#define DSA_MAX_SEGMENT_SIZE
Definition: dsa.c:101
dsa_segment_header * header
Definition: dsa.c:343
void dsm_pin_segment(dsm_segment *seg)
Definition: dsm.c:921
#define DSM_HANDLE_INVALID
Definition: dsm.h:23
#define DSA_INITIAL_SEGMENT_SIZE
Definition: dsa.c:68
void dsm_pin_mapping(dsm_segment *seg)
Definition: dsm.c:881
dsa_segment_index next
Definition: dsa.c:170
dsa_area_control * control
Definition: dsa.c:357
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE
Definition: dsa.c:77
size_t max_total_segment_size
Definition: dsa.c:313
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
Definition: freepage.c:183
dsm_segment * dsm_create(Size size, int flags)
Definition: dsm.c:487
int64 total_size
Definition: pg_checksums.c:68
#define FPM_PAGE_SIZE
Definition: freepage.h:30
void * dsm_segment_address(dsm_segment *seg)
Definition: dsm.c:1059
#define Assert(condition)
Definition: c.h:804
#define DSA_SEGMENT_INDEX_NONE
Definition: dsa.c:127
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:305
size_t dsa_segment_index
Definition: dsa.c:124
#define MAXALIGN(LEN)
Definition: c.h:757
dsa_pointer * pagemap
Definition: dsa.c:345
char * mapped_address
Definition: dsa.c:342
#define DSA_MAX_SEGMENTS
Definition: dsa.c:94
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:368
#define DSA_AREA_LOCK(area)
Definition: dsa.c:143
#define DSA_SEGMENT_HEADER_MAGIC
Definition: dsa.c:111
uint32 magic
Definition: dsa.c:154
size_t bin
Definition: dsa.c:172
dsa_handle handle
Definition: dsa.c:303
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ transfer_first_span()

static bool transfer_first_span ( dsa_area area,
dsa_area_pool pool,
int  fromclass,
int  toclass 
)
static

Definition at line 1397 of file dsa.c.

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, InvalidDsaPointer, dsa_area_span::nextspan, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by alloc_object(), and ensure_active_superblock().

1399 {
1400  dsa_pointer span_pointer;
1401  dsa_area_span *span;
1402  dsa_area_span *nextspan;
1403 
1404  /* Can't do it if source list is empty. */
1405  span_pointer = pool->spans[fromclass];
1406  if (!DsaPointerIsValid(span_pointer))
1407  return false;
1408 
1409  /* Remove span from head of source list. */
1410  span = dsa_get_address(area, span_pointer);
1411  pool->spans[fromclass] = span->nextspan;
1412  if (DsaPointerIsValid(span->nextspan))
1413  {
1414  nextspan = (dsa_area_span *)
1415  dsa_get_address(area, span->nextspan);
1416  nextspan->prevspan = InvalidDsaPointer;
1417  }
1418 
1419  /* Add span to head of target list. */
1420  span->nextspan = pool->spans[toclass];
1421  pool->spans[toclass] = span_pointer;
1422  if (DsaPointerIsValid(span->nextspan))
1423  {
1424  nextspan = (dsa_area_span *)
1425  dsa_get_address(area, span->nextspan);
1426  nextspan->prevspan = span_pointer;
1427  }
1428  span->fclass = toclass;
1429 
1430  return true;
1431 }
dsa_pointer nextspan
Definition: dsa.c:198
#define InvalidDsaPointer
Definition: dsa.h:78
uint64 dsa_pointer
Definition: dsa.h:62
uint16 fclass
Definition: dsa.c:206
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
dsa_pointer prevspan
Definition: dsa.c:197
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81

◆ unlink_segment()

static void unlink_segment ( dsa_area area,
dsa_segment_map segment_map 
)
static

Definition at line 1936 of file dsa.c.

References Assert, dsa_segment_header::bin, dsa_area::control, DSA_SEGMENT_INDEX_NONE, get_segment_by_index(), get_segment_index, dsa_segment_map::header, dsa_segment_header::next, next, dsa_segment_header::prev, and dsa_area_control::segment_bins.

Referenced by destroy_superblock(), and get_best_segment().

1937 {
1938  if (segment_map->header->prev != DSA_SEGMENT_INDEX_NONE)
1939  {
1940  dsa_segment_map *prev;
1941 
1942  prev = get_segment_by_index(area, segment_map->header->prev);
1943  prev->header->next = segment_map->header->next;
1944  }
1945  else
1946  {
1947  Assert(area->control->segment_bins[segment_map->header->bin] ==
1948  get_segment_index(area, segment_map));
1949  area->control->segment_bins[segment_map->header->bin] =
1950  segment_map->header->next;
1951  }
1952  if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
1953  {
1955 
1956  next = get_segment_by_index(area, segment_map->header->next);
1957  next->header->prev = segment_map->header->prev;
1958  }
1959 }
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
Definition: dsa.c:307
static int32 next
Definition: blutils.c:219
dsa_segment_index prev
Definition: dsa.c:164
dsa_segment_header * header
Definition: dsa.c:343
#define get_segment_index(area, segment_map_ptr)
Definition: dsa.c:381
dsa_segment_index next
Definition: dsa.c:170
dsa_area_control * control
Definition: dsa.c:357
#define Assert(condition)
Definition: c.h:804
#define DSA_SEGMENT_INDEX_NONE
Definition: dsa.c:127
size_t bin
Definition: dsa.c:172
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1722

◆ unlink_span()

static void unlink_span ( dsa_area area,
dsa_area_span span 
)
static

Definition at line 1864 of file dsa.c.

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, next, dsa_area_span::nextspan, dsa_area_span::pool, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by destroy_superblock(), and dsa_free().

1865 {
1866  if (DsaPointerIsValid(span->nextspan))
1867  {
1868  dsa_area_span *next = dsa_get_address(area, span->nextspan);
1869 
1870  next->prevspan = span->prevspan;
1871  }
1872  if (DsaPointerIsValid(span->prevspan))
1873  {
1874  dsa_area_span *prev = dsa_get_address(area, span->prevspan);
1875 
1876  prev->nextspan = span->nextspan;
1877  }
1878  else
1879  {
1880  dsa_area_pool *pool = dsa_get_address(area, span->pool);
1881 
1882  pool->spans[span->fclass] = span->nextspan;
1883  }
1884 }
dsa_pointer nextspan
Definition: dsa.c:198
static int32 next
Definition: blutils.c:219
uint16 fclass
Definition: dsa.c:206
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:932
dsa_pointer prevspan
Definition: dsa.c:197
dsa_pointer pool
Definition: dsa.c:196
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:290
#define DsaPointerIsValid(x)
Definition: dsa.h:81

Variable Documentation

◆ dsa_size_class_map

const uint8 dsa_size_class_map[]
static
Initial value:
= {
2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
}

Definition at line 259 of file dsa.c.

Referenced by dsa_allocate_extended().

◆ dsa_size_classes

const uint16 dsa_size_classes[]
static
Initial value:
= {
sizeof(dsa_area_span), 0,
8, 16, 24, 32, 40, 48, 56, 64,
80, 96, 112, 128,
160, 192, 224, 256,
320, 384, 448, 512,
640, 768, 896, 1024,
1280, 1560, 1816, 2048,
2616, 3120, 3640, 4096,
5456, 6552, 7280, 8192
}

Definition at line 236 of file dsa.c.

Referenced by alloc_object(), dsa_allocate_extended(), dsa_dump(), dsa_free(), ensure_active_superblock(), and init_span().