PostgreSQL Source Code  git master
dsa.c File Reference
#include "postgres.h"
#include "port/atomics.h"
#include "port/pg_bitutils.h"
#include "storage/dsm.h"
#include "storage/lwlock.h"
#include "utils/dsa.h"
#include "utils/freepage.h"
#include "utils/memutils.h"
#include "utils/resowner.h"
Include dependency graph for dsa.c:

Go to the source code of this file.

Data Structures

struct  dsa_segment_header
 
struct  dsa_area_span
 
struct  dsa_area_pool
 
struct  dsa_area_control
 
struct  dsa_segment_map
 
struct  dsa_area
 

Macros

#define DSA_NUM_SEGMENTS_AT_EACH_SIZE   2
 
#define DSA_MAX_SEGMENTS    Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
 
#define DSA_OFFSET_BITMASK   (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
 
#define DSA_PAGES_PER_SUPERBLOCK   16
 
#define DSA_SEGMENT_HEADER_MAGIC   0x0ce26608
 
#define DSA_MAKE_POINTER(segment_number, offset)    (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
 
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)   ((dp) >> DSA_OFFSET_WIDTH)
 
#define DSA_EXTRACT_OFFSET(dp)   ((dp) & DSA_OFFSET_BITMASK)
 
#define DSA_SEGMENT_INDEX_NONE   (~(dsa_segment_index)0)
 
#define DSA_NUM_SEGMENT_BINS   16
 
#define DSA_AREA_LOCK(area)   (&area->control->lock)
 
#define DSA_SCLASS_LOCK(area, sclass)   (&area->control->pools[sclass].lock)
 
#define NextFreeObjectIndex(object)   (* (uint16 *) (object))
 
#define DSA_NUM_SIZE_CLASSES   lengthof(dsa_size_classes)
 
#define DSA_SCLASS_BLOCK_OF_SPANS   0
 
#define DSA_SCLASS_SPAN_LARGE   1
 
#define DSA_SIZE_CLASS_MAP_QUANTUM   8
 
#define DSA_FULLNESS_CLASSES   4
 
#define DsaAreaPoolToDsaPointer(area, p)    DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
 
#define DSA_SPAN_NOTHING_FREE   ((uint16) -1)
 
#define DSA_SUPERBLOCK_SIZE   (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
 
#define get_segment_index(area, segment_map_ptr)    (segment_map_ptr - &area->segment_maps[0])
 

Typedefs

typedef size_t dsa_segment_index
 

Functions

static size_t contiguous_pages_to_segment_bin (size_t n)
 
static void init_span (dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
 
static bool transfer_first_span (dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
 
static dsa_pointer alloc_object (dsa_area *area, int size_class)
 
static bool ensure_active_superblock (dsa_area *area, dsa_area_pool *pool, int size_class)
 
static dsa_segment_mapget_segment_by_index (dsa_area *area, dsa_segment_index index)
 
static void destroy_superblock (dsa_area *area, dsa_pointer span_pointer)
 
static void unlink_span (dsa_area *area, dsa_area_span *span)
 
static void add_span_to_fullness_class (dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
 
static void unlink_segment (dsa_area *area, dsa_segment_map *segment_map)
 
static dsa_segment_mapget_best_segment (dsa_area *area, size_t npages)
 
static dsa_segment_mapmake_new_segment (dsa_area *area, size_t requested_pages)
 
static dsa_areacreate_internal (void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment, size_t init_segment_size, size_t max_segment_size)
 
static dsa_areaattach_internal (void *place, dsm_segment *segment, dsa_handle handle)
 
static void check_for_freed_segments (dsa_area *area)
 
static void check_for_freed_segments_locked (dsa_area *area)
 
static void rebin_segment (dsa_area *area, dsa_segment_map *segment_map)
 
dsa_areadsa_create_ext (int tranche_id, size_t init_segment_size, size_t max_segment_size)
 
dsa_areadsa_create_in_place_ext (void *place, size_t size, int tranche_id, dsm_segment *segment, size_t init_segment_size, size_t max_segment_size)
 
dsa_handle dsa_get_handle (dsa_area *area)
 
dsa_areadsa_attach (dsa_handle handle)
 
dsa_areadsa_attach_in_place (void *place, dsm_segment *segment)
 
void dsa_on_dsm_detach_release_in_place (dsm_segment *segment, Datum place)
 
void dsa_on_shmem_exit_release_in_place (int code, Datum place)
 
void dsa_release_in_place (void *place)
 
void dsa_pin_mapping (dsa_area *area)
 
dsa_pointer dsa_allocate_extended (dsa_area *area, size_t size, int flags)
 
void dsa_free (dsa_area *area, dsa_pointer dp)
 
void * dsa_get_address (dsa_area *area, dsa_pointer dp)
 
void dsa_pin (dsa_area *area)
 
void dsa_unpin (dsa_area *area)
 
void dsa_set_size_limit (dsa_area *area, size_t limit)
 
size_t dsa_get_total_size (dsa_area *area)
 
void dsa_trim (dsa_area *area)
 
void dsa_dump (dsa_area *area)
 
size_t dsa_minimum_size (void)
 
void dsa_detach (dsa_area *area)
 

Variables

static const uint16 dsa_size_classes []
 
static const uint8 dsa_size_class_map []
 

Macro Definition Documentation

◆ DSA_AREA_LOCK

#define DSA_AREA_LOCK (   area)    (&area->control->lock)

Definition at line 132 of file dsa.c.

◆ DSA_EXTRACT_OFFSET

#define DSA_EXTRACT_OFFSET (   dp)    ((dp) & DSA_OFFSET_BITMASK)

Definition at line 99 of file dsa.c.

◆ DSA_EXTRACT_SEGMENT_NUMBER

#define DSA_EXTRACT_SEGMENT_NUMBER (   dp)    ((dp) >> DSA_OFFSET_WIDTH)

Definition at line 96 of file dsa.c.

◆ DSA_FULLNESS_CLASSES

#define DSA_FULLNESS_CLASSES   4

Definition at line 266 of file dsa.c.

◆ DSA_MAKE_POINTER

#define DSA_MAKE_POINTER (   segment_number,
  offset 
)     (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))

Definition at line 92 of file dsa.c.

◆ DSA_MAX_SEGMENTS

#define DSA_MAX_SEGMENTS    Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))

Definition at line 75 of file dsa.c.

◆ DSA_NUM_SEGMENT_BINS

#define DSA_NUM_SEGMENT_BINS   16

Definition at line 111 of file dsa.c.

◆ DSA_NUM_SEGMENTS_AT_EACH_SIZE

#define DSA_NUM_SEGMENTS_AT_EACH_SIZE   2

Definition at line 69 of file dsa.c.

◆ DSA_NUM_SIZE_CLASSES

#define DSA_NUM_SIZE_CLASSES   lengthof(dsa_size_classes)

Definition at line 236 of file dsa.c.

◆ DSA_OFFSET_BITMASK

#define DSA_OFFSET_BITMASK   (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)

Definition at line 79 of file dsa.c.

◆ DSA_PAGES_PER_SUPERBLOCK

#define DSA_PAGES_PER_SUPERBLOCK   16

Definition at line 82 of file dsa.c.

◆ DSA_SCLASS_BLOCK_OF_SPANS

#define DSA_SCLASS_BLOCK_OF_SPANS   0

Definition at line 239 of file dsa.c.

◆ DSA_SCLASS_LOCK

#define DSA_SCLASS_LOCK (   area,
  sclass 
)    (&area->control->pools[sclass].lock)

Definition at line 133 of file dsa.c.

◆ DSA_SCLASS_SPAN_LARGE

#define DSA_SCLASS_SPAN_LARGE   1

Definition at line 240 of file dsa.c.

◆ DSA_SEGMENT_HEADER_MAGIC

#define DSA_SEGMENT_HEADER_MAGIC   0x0ce26608

Definition at line 89 of file dsa.c.

◆ DSA_SEGMENT_INDEX_NONE

#define DSA_SEGMENT_INDEX_NONE   (~(dsa_segment_index)0)

Definition at line 105 of file dsa.c.

◆ DSA_SIZE_CLASS_MAP_QUANTUM

#define DSA_SIZE_CLASS_MAP_QUANTUM   8

Definition at line 258 of file dsa.c.

◆ DSA_SPAN_NOTHING_FREE

#define DSA_SPAN_NOTHING_FREE   ((uint16) -1)

Definition at line 375 of file dsa.c.

◆ DSA_SUPERBLOCK_SIZE

#define DSA_SUPERBLOCK_SIZE   (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)

Definition at line 376 of file dsa.c.

◆ DsaAreaPoolToDsaPointer

#define DsaAreaPoolToDsaPointer (   area,
 
)     DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)

Definition at line 322 of file dsa.c.

◆ get_segment_index

#define get_segment_index (   area,
  segment_map_ptr 
)     (segment_map_ptr - &area->segment_maps[0])

Definition at line 379 of file dsa.c.

◆ NextFreeObjectIndex

#define NextFreeObjectIndex (   object)    (* (uint16 *) (object))

Definition at line 202 of file dsa.c.

Typedef Documentation

◆ dsa_segment_index

typedef size_t dsa_segment_index

Definition at line 102 of file dsa.c.

Function Documentation

◆ add_span_to_fullness_class()

static void add_span_to_fullness_class ( dsa_area area,
dsa_area_span span,
dsa_pointer  span_pointer,
int  fclass 
)
static

Definition at line 1929 of file dsa.c.

1932 {
1933  dsa_area_pool *pool = dsa_get_address(area, span->pool);
1934 
1935  if (DsaPointerIsValid(pool->spans[fclass]))
1936  {
1937  dsa_area_span *head = dsa_get_address(area,
1938  pool->spans[fclass]);
1939 
1940  head->prevspan = span_pointer;
1941  }
1942  span->prevspan = InvalidDsaPointer;
1943  span->nextspan = pool->spans[fclass];
1944  pool->spans[fclass] = span_pointer;
1945  span->fclass = fclass;
1946 }
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:942
#define InvalidDsaPointer
Definition: dsa.h:78
#define DsaPointerIsValid(x)
Definition: dsa.h:106
dsa_pointer spans[DSA_FULLNESS_CLASSES]
Definition: dsa.c:279
dsa_pointer nextspan
Definition: dsa.c:187
uint16 fclass
Definition: dsa.c:195
dsa_pointer prevspan
Definition: dsa.c:186
dsa_pointer pool
Definition: dsa.c:185

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, InvalidDsaPointer, dsa_area_span::nextspan, dsa_area_span::pool, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by dsa_free().

◆ alloc_object()

static dsa_pointer alloc_object ( dsa_area area,
int  size_class 
)
inlinestatic

Definition at line 1472 of file dsa.c.

1473 {
1474  dsa_area_pool *pool = &area->control->pools[size_class];
1475  dsa_area_span *span;
1476  dsa_pointer block;
1477  dsa_pointer result;
1478  char *object;
1479  size_t size;
1480 
1481  /*
1482  * Even though ensure_active_superblock can in turn call alloc_object if
1483  * it needs to allocate a new span, that's always from a different pool,
1484  * and the order of lock acquisition is always the same, so it's OK that
1485  * we hold this lock for the duration of this function.
1486  */
1487  Assert(!LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1488  LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
1489 
1490  /*
1491  * If there's no active superblock, we must successfully obtain one or
1492  * fail the request.
1493  */
1494  if (!DsaPointerIsValid(pool->spans[1]) &&
1495  !ensure_active_superblock(area, pool, size_class))
1496  {
1497  result = InvalidDsaPointer;
1498  }
1499  else
1500  {
1501  /*
1502  * There should be a block in fullness class 1 at this point, and it
1503  * should never be completely full. Thus we can either pop an object
1504  * from the free list or, failing that, initialize a new object.
1505  */
1506  Assert(DsaPointerIsValid(pool->spans[1]));
1507  span = (dsa_area_span *)
1508  dsa_get_address(area, pool->spans[1]);
1509  Assert(span->nallocatable > 0);
1510  block = span->start;
1511  Assert(size_class < DSA_NUM_SIZE_CLASSES);
1512  size = dsa_size_classes[size_class];
1513  if (span->firstfree != DSA_SPAN_NOTHING_FREE)
1514  {
1515  result = block + span->firstfree * size;
1516  object = dsa_get_address(area, result);
1517  span->firstfree = NextFreeObjectIndex(object);
1518  }
1519  else
1520  {
1521  result = block + span->ninitialized * size;
1522  ++span->ninitialized;
1523  }
1524  --span->nallocatable;
1525 
1526  /* If it's now full, move it to the highest-numbered fullness class. */
1527  if (span->nallocatable == 0)
1528  transfer_first_span(area, pool, 1, DSA_FULLNESS_CLASSES - 1);
1529  }
1530 
1531  Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1532  LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
1533 
1534  return result;
1535 }
#define Assert(condition)
Definition: c.h:858
static const uint16 dsa_size_classes[]
Definition: dsa.c:225
static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)
Definition: dsa.c:1560
#define DSA_SPAN_NOTHING_FREE
Definition: dsa.c:375
#define DSA_SCLASS_LOCK(area, sclass)
Definition: dsa.c:133
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
Definition: dsa.c:1432
#define DSA_NUM_SIZE_CLASSES
Definition: dsa.c:236
#define NextFreeObjectIndex(object)
Definition: dsa.c:202
#define DSA_FULLNESS_CLASSES
Definition: dsa.c:266
uint64 dsa_pointer
Definition: dsa.h:62
bool LWLockHeldByMe(LWLock *lock)
Definition: lwlock.c:1895
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1170
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1783
@ LW_EXCLUSIVE
Definition: lwlock.h:114
static pg_noinline void Size size
Definition: slab.c:607
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
Definition: dsa.c:298
dsa_pointer start
Definition: dsa.c:188
uint16 nallocatable
Definition: dsa.c:192
uint16 ninitialized
Definition: dsa.c:191
uint16 firstfree
Definition: dsa.c:193
dsa_area_control * control
Definition: dsa.c:350

References Assert, dsa_area::control, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DsaPointerIsValid, ensure_active_superblock(), dsa_area_span::firstfree, InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::ninitialized, dsa_area_control::pools, size, dsa_area_pool::spans, dsa_area_span::start, and transfer_first_span().

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ attach_internal()

static dsa_area * attach_internal ( void *  place,
dsm_segment segment,
dsa_handle  handle 
)
static

Definition at line 1326 of file dsa.c.

1327 {
1328  dsa_area_control *control;
1329  dsa_area *area;
1330  dsa_segment_map *segment_map;
1331 
1332  control = (dsa_area_control *) place;
1333  Assert(control->handle == handle);
1334  Assert(control->segment_handles[0] == handle);
1335  Assert(control->segment_header.magic ==
1336  (DSA_SEGMENT_HEADER_MAGIC ^ handle ^ 0));
1337 
1338  /* Build the backend-local area object. */
1339  area = palloc(sizeof(dsa_area));
1340  area->control = control;
1342  memset(&area->segment_maps[0], 0,
1343  sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
1344  area->high_segment_index = 0;
1345 
1346  /* Set up the segment map for this process's mapping. */
1347  segment_map = &area->segment_maps[0];
1348  segment_map->segment = segment; /* NULL for in-place */
1349  segment_map->mapped_address = place;
1350  segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
1351  segment_map->fpm = (FreePageManager *)
1352  (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)));
1353  segment_map->pagemap = (dsa_pointer *)
1354  (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)) +
1355  MAXALIGN(sizeof(FreePageManager)));
1356 
1357  /* Bump the reference count. */
1359  if (control->refcnt == 0)
1360  {
1361  /* We can't attach to a DSA area that has already been destroyed. */
1362  ereport(ERROR,
1363  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1364  errmsg("could not attach to dynamic shared area")));
1365  }
1366  ++control->refcnt;
1369 
1370  return area;
1371 }
#define MAXALIGN(LEN)
Definition: c.h:811
#define DSA_AREA_LOCK(area)
Definition: dsa.c:132
#define DSA_SEGMENT_HEADER_MAGIC
Definition: dsa.c:89
#define DSA_MAX_SEGMENTS
Definition: dsa.c:75
int errcode(int sqlerrcode)
Definition: elog.c:857
int errmsg(const char *fmt,...)
Definition: elog.c:1070
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
void * palloc(Size size)
Definition: mcxt.c:1316
ResourceOwner CurrentResourceOwner
Definition: resowner.c:165
dsa_segment_header segment_header
Definition: dsa.c:290
size_t freed_segment_counter
Definition: dsa.c:314
int refcnt
Definition: dsa.c:310
dsa_handle handle
Definition: dsa.c:292
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
Definition: dsa.c:294
Definition: dsa.c:348
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
Definition: dsa.c:366
dsa_segment_index high_segment_index
Definition: dsa.c:369
size_t freed_segment_counter
Definition: dsa.c:372
ResourceOwner resowner
Definition: dsa.c:358
uint32 magic
Definition: dsa.c:143
dsa_segment_header * header
Definition: dsa.c:336
FreePageManager * fpm
Definition: dsa.c:337
dsm_segment * segment
Definition: dsa.c:334
dsa_pointer * pagemap
Definition: dsa.c:338
char * mapped_address
Definition: dsa.c:335

References Assert, dsa_area::control, CurrentResourceOwner, DSA_AREA_LOCK, DSA_MAX_SEGMENTS, DSA_SEGMENT_HEADER_MAGIC, ereport, errcode(), errmsg(), ERROR, dsa_segment_map::fpm, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_segment_map::mapped_address, MAXALIGN, dsa_segment_map::pagemap, palloc(), dsa_area_control::refcnt, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area_control::segment_header, and dsa_area::segment_maps.

Referenced by dsa_attach(), and dsa_attach_in_place().

◆ check_for_freed_segments()

static void check_for_freed_segments ( dsa_area area)
static

Definition at line 2252 of file dsa.c.

2253 {
2254  size_t freed_segment_counter;
2255 
2256  /*
2257  * Any other process that has freed a segment has incremented
2258  * freed_segment_counter while holding an LWLock, and that must precede
2259  * any backend creating a new segment in the same slot while holding an
2260  * LWLock, and that must precede the creation of any dsa_pointer pointing
2261  * into the new segment which might reach us here, and the caller must
2262  * have sent the dsa_pointer to this process using appropriate memory
2263  * synchronization (some kind of locking or atomic primitive or system
2264  * call). So all we need to do on the reading side is ask for the load of
2265  * freed_segment_counter to follow the caller's load of the dsa_pointer it
2266  * has, and we can be sure to detect any segments that had been freed as
2267  * of the time that the dsa_pointer reached this process.
2268  */
2269  pg_read_barrier();
2270  freed_segment_counter = area->control->freed_segment_counter;
2271  if (unlikely(area->freed_segment_counter != freed_segment_counter))
2272  {
2273  /* Check all currently mapped segments to find what's been freed. */
2277  }
2278 }
#define pg_read_barrier()
Definition: atomics.h:151
#define unlikely(x)
Definition: c.h:311
static void check_for_freed_segments_locked(dsa_area *area)
Definition: dsa.c:2288

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), pg_read_barrier, and unlikely.

Referenced by dsa_free(), and dsa_get_address().

◆ check_for_freed_segments_locked()

static void check_for_freed_segments_locked ( dsa_area area)
static

Definition at line 2288 of file dsa.c.

2289 {
2290  size_t freed_segment_counter;
2291  int i;
2292 
2294  freed_segment_counter = area->control->freed_segment_counter;
2295  if (unlikely(area->freed_segment_counter != freed_segment_counter))
2296  {
2297  for (i = 0; i <= area->high_segment_index; ++i)
2298  {
2299  if (area->segment_maps[i].header != NULL &&
2300  area->segment_maps[i].header->freed)
2301  {
2302  dsm_detach(area->segment_maps[i].segment);
2303  area->segment_maps[i].segment = NULL;
2304  area->segment_maps[i].header = NULL;
2305  area->segment_maps[i].mapped_address = NULL;
2306  }
2307  }
2308  area->freed_segment_counter = freed_segment_counter;
2309  }
2310 }
void dsm_detach(dsm_segment *seg)
Definition: dsm.c:803
int i
Definition: isn.c:73

References Assert, dsa_area::control, DSA_AREA_LOCK, dsm_detach(), dsa_segment_header::freed, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_segment_map::header, dsa_area::high_segment_index, i, LWLockHeldByMe(), dsa_segment_map::mapped_address, dsa_segment_map::segment, dsa_area::segment_maps, and unlikely.

Referenced by check_for_freed_segments(), destroy_superblock(), dsa_dump(), and get_best_segment().

◆ contiguous_pages_to_segment_bin()

static size_t contiguous_pages_to_segment_bin ( size_t  n)
inlinestatic

Definition at line 119 of file dsa.c.

120 {
121  size_t bin;
122 
123  if (n == 0)
124  bin = 0;
125  else
126  bin = pg_leftmost_one_pos_size_t(n) + 1;
127 
128  return Min(bin, DSA_NUM_SEGMENT_BINS - 1);
129 }
#define Min(x, y)
Definition: c.h:1004
#define DSA_NUM_SEGMENT_BINS
Definition: dsa.c:111
#define pg_leftmost_one_pos_size_t
Definition: pg_bitutils.h:416

References DSA_NUM_SEGMENT_BINS, Min, and pg_leftmost_one_pos_size_t.

Referenced by create_internal(), get_best_segment(), make_new_segment(), and rebin_segment().

◆ create_internal()

static dsa_area * create_internal ( void *  place,
size_t  size,
int  tranche_id,
dsm_handle  control_handle,
dsm_segment control_segment,
size_t  init_segment_size,
size_t  max_segment_size 
)
static

Definition at line 1218 of file dsa.c.

1223 {
1224  dsa_area_control *control;
1225  dsa_area *area;
1226  dsa_segment_map *segment_map;
1227  size_t usable_pages;
1228  size_t total_pages;
1229  size_t metadata_bytes;
1230  int i;
1231 
1232  /* Check the initial and maximum block sizes */
1233  Assert(init_segment_size >= DSA_MIN_SEGMENT_SIZE);
1234  Assert(max_segment_size >= init_segment_size);
1235  Assert(max_segment_size <= DSA_MAX_SEGMENT_SIZE);
1236 
1237  /* Sanity check on the space we have to work in. */
1238  if (size < dsa_minimum_size())
1239  elog(ERROR, "dsa_area space must be at least %zu, but %zu provided",
1240  dsa_minimum_size(), size);
1241 
1242  /* Now figure out how much space is usable */
1243  total_pages = size / FPM_PAGE_SIZE;
1244  metadata_bytes =
1245  MAXALIGN(sizeof(dsa_area_control)) +
1246  MAXALIGN(sizeof(FreePageManager)) +
1247  total_pages * sizeof(dsa_pointer);
1248  /* Add padding up to next page boundary. */
1249  if (metadata_bytes % FPM_PAGE_SIZE != 0)
1250  metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
1251  Assert(metadata_bytes <= size);
1252  usable_pages = (size - metadata_bytes) / FPM_PAGE_SIZE;
1253 
1254  /*
1255  * Initialize the dsa_area_control object located at the start of the
1256  * space.
1257  */
1258  control = (dsa_area_control *) place;
1259  memset(place, 0, sizeof(*control));
1260  control->segment_header.magic =
1261  DSA_SEGMENT_HEADER_MAGIC ^ control_handle ^ 0;
1264  control->segment_header.usable_pages = usable_pages;
1265  control->segment_header.freed = false;
1266  control->segment_header.size = size;
1267  control->handle = control_handle;
1268  control->init_segment_size = init_segment_size;
1269  control->max_segment_size = max_segment_size;
1270  control->max_total_segment_size = (size_t) -1;
1271  control->total_segment_size = size;
1272  control->segment_handles[0] = control_handle;
1273  for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
1275  control->refcnt = 1;
1276  control->lwlock_tranche_id = tranche_id;
1277 
1278  /*
1279  * Create the dsa_area object that this backend will use to access the
1280  * area. Other backends will need to obtain their own dsa_area object by
1281  * attaching.
1282  */
1283  area = palloc(sizeof(dsa_area));
1284  area->control = control;
1286  memset(area->segment_maps, 0, sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
1287  area->high_segment_index = 0;
1288  area->freed_segment_counter = 0;
1289  LWLockInitialize(&control->lock, control->lwlock_tranche_id);
1290  for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
1292  control->lwlock_tranche_id);
1293 
1294  /* Set up the segment map for this process's mapping. */
1295  segment_map = &area->segment_maps[0];
1296  segment_map->segment = control_segment;
1297  segment_map->mapped_address = place;
1298  segment_map->header = (dsa_segment_header *) place;
1299  segment_map->fpm = (FreePageManager *)
1300  (segment_map->mapped_address +
1301  MAXALIGN(sizeof(dsa_area_control)));
1302  segment_map->pagemap = (dsa_pointer *)
1303  (segment_map->mapped_address +
1304  MAXALIGN(sizeof(dsa_area_control)) +
1305  MAXALIGN(sizeof(FreePageManager)));
1306 
1307  /* Set up the free page map. */
1308  FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
1309  /* There can be 0 usable pages if size is dsa_minimum_size(). */
1310 
1311  if (usable_pages > 0)
1312  FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
1313  usable_pages);
1314 
1315  /* Put this segment into the appropriate bin. */
1316  control->segment_bins[contiguous_pages_to_segment_bin(usable_pages)] = 0;
1317  segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
1318 
1319  return area;
1320 }
#define DSA_SEGMENT_INDEX_NONE
Definition: dsa.c:105
static size_t contiguous_pages_to_segment_bin(size_t n)
Definition: dsa.c:119
size_t dsa_minimum_size(void)
Definition: dsa.c:1196
#define DSA_MIN_SEGMENT_SIZE
Definition: dsa.h:100
#define DSA_MAX_SEGMENT_SIZE
Definition: dsa.h:103
#define elog(elevel,...)
Definition: elog.h:224
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
Definition: freepage.c:379
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
Definition: freepage.c:183
#define FPM_PAGE_SIZE
Definition: freepage.h:30
void LWLockInitialize(LWLock *lock, int tranche_id)
Definition: lwlock.c:709
size_t init_segment_size
Definition: dsa.c:300
size_t total_segment_size
Definition: dsa.c:304
int lwlock_tranche_id
Definition: dsa.c:316
size_t max_segment_size
Definition: dsa.c:302
size_t max_total_segment_size
Definition: dsa.c:306
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
Definition: dsa.c:296
LWLock lock
Definition: dsa.c:318
size_t size
Definition: dsa.c:147
dsa_segment_index next
Definition: dsa.c:159
dsa_segment_index prev
Definition: dsa.c:153
size_t usable_pages
Definition: dsa.c:145
size_t bin
Definition: dsa.c:161

References Assert, dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, CurrentResourceOwner, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_MIN_SEGMENT_SIZE, dsa_minimum_size(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, elog, ERROR, dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area::freed_segment_counter, FreePageManagerInitialize(), FreePageManagerPut(), dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, i, dsa_area_control::init_segment_size, dsa_area_control::lock, dsa_area_control::lwlock_tranche_id, LWLockInitialize(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area_control::max_segment_size, dsa_area_control::max_total_segment_size, MAXALIGN, dsa_segment_header::next, dsa_segment_map::pagemap, palloc(), dsa_segment_header::prev, dsa_area_control::refcnt, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area_control::segment_header, dsa_area::segment_maps, dsa_segment_header::size, size, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

Referenced by dsa_create_ext(), and dsa_create_in_place_ext().

◆ destroy_superblock()

static void destroy_superblock ( dsa_area area,
dsa_pointer  span_pointer 
)
static

Definition at line 1837 of file dsa.c.

1838 {
1839  dsa_area_span *span = dsa_get_address(area, span_pointer);
1840  int size_class = span->size_class;
1841  dsa_segment_map *segment_map;
1842 
1843 
1844  /* Remove it from its fullness class list. */
1845  unlink_span(area, span);
1846 
1847  /*
1848  * Note: Here we acquire the area lock while we already hold a per-pool
1849  * lock. We never hold the area lock and then take a pool lock, or we
1850  * could deadlock.
1851  */
1854  segment_map =
1856  FreePageManagerPut(segment_map->fpm,
1858  span->npages);
1859  /* Check if the segment is now entirely free. */
1860  if (fpm_largest(segment_map->fpm) == segment_map->header->usable_pages)
1861  {
1862  dsa_segment_index index = get_segment_index(area, segment_map);
1863 
1864  /* If it's not the segment with extra control data, free it. */
1865  if (index != 0)
1866  {
1867  /*
1868  * Give it back to the OS, and allow other backends to detect that
1869  * they need to detach.
1870  */
1871  unlink_segment(area, segment_map);
1872  segment_map->header->freed = true;
1874  segment_map->header->size);
1875  area->control->total_segment_size -=
1876  segment_map->header->size;
1878  dsm_detach(segment_map->segment);
1880  ++area->control->freed_segment_counter;
1881  segment_map->segment = NULL;
1882  segment_map->header = NULL;
1883  segment_map->mapped_address = NULL;
1884  }
1885  }
1886 
1887  /* Move segment to appropriate bin if necessary. */
1888  if (segment_map->header != NULL)
1889  rebin_segment(area, segment_map);
1890 
1892 
1893  /*
1894  * Span-of-spans blocks store the span which describes them within the
1895  * block itself, so freeing the storage implicitly frees the descriptor
1896  * also. If this is a block of any other type, we need to separately free
1897  * the span object also. This recursive call to dsa_free will acquire the
1898  * span pool's lock. We can't deadlock because the acquisition order is
1899  * always some other pool and then the span pool.
1900  */
1901  if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
1902  dsa_free(area, span_pointer);
1903 }
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
Definition: dsa.c:1978
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
Definition: dsa.c:96
#define get_segment_index(area, segment_map_ptr)
Definition: dsa.c:379
#define DSA_EXTRACT_OFFSET(dp)
Definition: dsa.c:99
size_t dsa_segment_index
Definition: dsa.c:102
static void rebin_segment(dsa_area *area, dsa_segment_map *segment_map)
Definition: dsa.c:2316
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
Definition: dsa.c:1757
#define DSA_SCLASS_BLOCK_OF_SPANS
Definition: dsa.c:239
static void unlink_span(dsa_area *area, dsa_area_span *span)
Definition: dsa.c:1906
void dsa_free(dsa_area *area, dsa_pointer dp)
Definition: dsa.c:826
dsm_handle dsm_segment_handle(dsm_segment *seg)
Definition: dsm.c:1123
void dsm_unpin_segment(dsm_handle handle)
Definition: dsm.c:988
#define DSM_HANDLE_INVALID
Definition: dsm_impl.h:58
#define fpm_largest(fpm)
Definition: freepage.h:88
uint16 size_class
Definition: dsa.c:190
size_t npages
Definition: dsa.c:189
Definition: type.h:95

References Assert, check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, dsm_detach(), DSM_HANDLE_INVALID, dsm_segment_handle(), dsm_unpin_segment(), dsa_segment_map::fpm, fpm_largest, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area_control::freed_segment_counter, FreePageManagerPut(), get_segment_by_index(), get_segment_index, dsa_segment_map::header, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_span::npages, rebin_segment(), dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_segment_header::size, dsa_area_span::size_class, dsa_area_span::start, dsa_area_control::total_segment_size, unlink_segment(), unlink_span(), and dsa_segment_header::usable_pages.

Referenced by dsa_free(), and dsa_trim().

◆ dsa_allocate_extended()

dsa_pointer dsa_allocate_extended ( dsa_area area,
size_t  size,
int  flags 
)

Definition at line 671 of file dsa.c.

672 {
673  uint16 size_class;
674  dsa_pointer start_pointer;
675  dsa_segment_map *segment_map;
676  dsa_pointer result;
677 
678  Assert(size > 0);
679 
680  /* Sanity check on huge individual allocation size. */
681  if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
682  ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
683  elog(ERROR, "invalid DSA memory alloc request size %zu", size);
684 
685  /*
686  * If bigger than the largest size class, just grab a run of pages from
687  * the free page manager, instead of allocating an object from a pool.
688  * There will still be a span, but it's a special class of span that
689  * manages this whole allocation and simply gives all pages back to the
690  * free page manager when dsa_free is called.
691  */
693  {
694  size_t npages = fpm_size_to_pages(size);
695  size_t first_page;
696  dsa_pointer span_pointer;
698 
699  /* Obtain a span object. */
700  span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
701  if (!DsaPointerIsValid(span_pointer))
702  {
703  /* Raise error unless asked not to. */
704  if ((flags & DSA_ALLOC_NO_OOM) == 0)
705  ereport(ERROR,
706  (errcode(ERRCODE_OUT_OF_MEMORY),
707  errmsg("out of memory"),
708  errdetail("Failed on DSA request of size %zu.",
709  size)));
710  return InvalidDsaPointer;
711  }
712 
714 
715  /* Find a segment from which to allocate. */
716  segment_map = get_best_segment(area, npages);
717  if (segment_map == NULL)
718  segment_map = make_new_segment(area, npages);
719  if (segment_map == NULL)
720  {
721  /* Can't make any more segments: game over. */
723  dsa_free(area, span_pointer);
724 
725  /* Raise error unless asked not to. */
726  if ((flags & DSA_ALLOC_NO_OOM) == 0)
727  ereport(ERROR,
728  (errcode(ERRCODE_OUT_OF_MEMORY),
729  errmsg("out of memory"),
730  errdetail("Failed on DSA request of size %zu.",
731  size)));
732  return InvalidDsaPointer;
733  }
734 
735  /*
736  * Ask the free page manager for a run of pages. This should always
737  * succeed, since both get_best_segment and make_new_segment should
738  * only return a non-NULL pointer if it actually contains enough
739  * contiguous freespace. If it does fail, something in our backend
740  * private state is out of whack, so use FATAL to kill the process.
741  */
742  if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
743  elog(FATAL,
744  "dsa_allocate could not find %zu free pages", npages);
746 
747  start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
748  first_page * FPM_PAGE_SIZE);
749 
750  /* Initialize span and pagemap. */
752  LW_EXCLUSIVE);
753  init_span(area, span_pointer, pool, start_pointer, npages,
755  segment_map->pagemap[first_page] = span_pointer;
757 
758  /* Zero-initialize the memory if requested. */
759  if ((flags & DSA_ALLOC_ZERO) != 0)
760  memset(dsa_get_address(area, start_pointer), 0, size);
761 
762  return start_pointer;
763  }
764 
765  /* Map allocation to a size class. */
767  {
768  int mapidx;
769 
770  /* For smaller sizes we have a lookup table... */
771  mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
773  size_class = dsa_size_class_map[mapidx];
774  }
775  else
776  {
777  uint16 min;
778  uint16 max;
779 
780  /* ... and for the rest we search by binary chop. */
782  max = lengthof(dsa_size_classes) - 1;
783 
784  while (min < max)
785  {
786  uint16 mid = (min + max) / 2;
787  uint16 class_size = dsa_size_classes[mid];
788 
789  if (class_size < size)
790  min = mid + 1;
791  else
792  max = mid;
793  }
794 
795  size_class = min;
796  }
797  Assert(size <= dsa_size_classes[size_class]);
798  Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
799 
800  /* Attempt to allocate an object from the appropriate pool. */
801  result = alloc_object(area, size_class);
802 
803  /* Check for failure to allocate. */
804  if (!DsaPointerIsValid(result))
805  {
806  /* Raise error unless asked not to. */
807  if ((flags & DSA_ALLOC_NO_OOM) == 0)
808  ereport(ERROR,
809  (errcode(ERRCODE_OUT_OF_MEMORY),
810  errmsg("out of memory"),
811  errdetail("Failed on DSA request of size %zu.", size)));
812  return InvalidDsaPointer;
813  }
814 
815  /* Zero-initialize the memory if requested. */
816  if ((flags & DSA_ALLOC_ZERO) != 0)
817  memset(dsa_get_address(area, result), 0, size);
818 
819  return result;
820 }
unsigned short uint16
Definition: c.h:505
#define lengthof(array)
Definition: c.h:788
#define DSA_MAKE_POINTER(segment_number, offset)
Definition: dsa.c:92
static dsa_pointer alloc_object(dsa_area *area, int size_class)
Definition: dsa.c:1472
#define DSA_SIZE_CLASS_MAP_QUANTUM
Definition: dsa.c:258
static const uint8 dsa_size_class_map[]
Definition: dsa.c:248
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
Definition: dsa.c:2081
#define DSA_SCLASS_SPAN_LARGE
Definition: dsa.c:240
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
Definition: dsa.c:1377
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
Definition: dsa.c:2010
#define DSA_ALLOC_NO_OOM
Definition: dsa.h:74
#define DSA_ALLOC_HUGE
Definition: dsa.h:73
#define DSA_ALLOC_ZERO
Definition: dsa.h:75
int errdetail(const char *fmt,...)
Definition: elog.c:1203
#define FATAL
Definition: elog.h:41
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
Definition: freepage.c:210
#define fpm_size_to_pages(sz)
Definition: freepage.h:74
#define AllocHugeSizeIsValid(size)
Definition: memutils.h:49
#define AllocSizeIsValid(size)
Definition: memutils.h:42

References alloc_object(), AllocHugeSizeIsValid, AllocSizeIsValid, Assert, dsa_area::control, DSA_ALLOC_HUGE, DSA_ALLOC_NO_OOM, DSA_ALLOC_ZERO, DSA_AREA_LOCK, dsa_free(), dsa_get_address(), DSA_MAKE_POINTER, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_class_map, DSA_SIZE_CLASS_MAP_QUANTUM, dsa_size_classes, DsaPointerIsValid, elog, ereport, errcode(), errdetail(), errmsg(), ERROR, FATAL, dsa_segment_map::fpm, FPM_PAGE_SIZE, fpm_size_to_pages, FreePageManagerGet(), get_best_segment(), get_segment_index, init_span(), InvalidDsaPointer, lengthof, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), make_new_segment(), dsa_segment_map::pagemap, dsa_area_control::pools, and size.

Referenced by dshash_create(), and pagetable_allocate().

◆ dsa_attach()

dsa_area* dsa_attach ( dsa_handle  handle)

Definition at line 510 of file dsa.c.

511 {
512  dsm_segment *segment;
513  dsa_area *area;
514 
515  /*
516  * An area handle is really a DSM segment handle for the first segment, so
517  * we go ahead and attach to that.
518  */
519  segment = dsm_attach(handle);
520  if (segment == NULL)
521  ereport(ERROR,
522  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
523  errmsg("could not attach to dynamic shared area")));
524 
525  area = attach_internal(dsm_segment_address(segment), segment, handle);
526 
527  /* Clean up when the control segment detaches. */
530 
531  return area;
532 }
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
Definition: dsa.c:576
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
Definition: dsa.c:1326
void * dsm_segment_address(dsm_segment *seg)
Definition: dsm.c:1095
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
Definition: dsm.c:1132
dsm_segment * dsm_attach(dsm_handle h)
Definition: dsm.c:665
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:322

References attach_internal(), dsa_on_dsm_detach_release_in_place(), dsm_attach(), dsm_segment_address(), ereport, errcode(), errmsg(), ERROR, on_dsm_detach(), and PointerGetDatum().

Referenced by init_dsm_registry(), logicalrep_launcher_attach_dshmem(), and TidStoreAttach().

◆ dsa_attach_in_place()

dsa_area* dsa_attach_in_place ( void *  place,
dsm_segment segment 
)

Definition at line 545 of file dsa.c.

546 {
547  dsa_area *area;
548 
549  area = attach_internal(place, NULL, DSA_HANDLE_INVALID);
550 
551  /*
552  * Clean up when the control segment detaches, if a containing DSM segment
553  * was provided.
554  */
555  if (segment != NULL)
557  PointerGetDatum(place));
558 
559  return area;
560 }
#define DSA_HANDLE_INVALID
Definition: dsa.h:139

References attach_internal(), DSA_HANDLE_INVALID, dsa_on_dsm_detach_release_in_place(), on_dsm_detach(), and PointerGetDatum().

Referenced by AttachSession(), ParallelQueryMain(), and pgstat_attach_shmem().

◆ dsa_create_ext()

dsa_area* dsa_create_ext ( int  tranche_id,
size_t  init_segment_size,
size_t  max_segment_size 
)

Definition at line 421 of file dsa.c.

422 {
423  dsm_segment *segment;
424  dsa_area *area;
425 
426  /*
427  * Create the DSM segment that will hold the shared control object and the
428  * first segment of usable space.
429  */
430  segment = dsm_create(init_segment_size, 0);
431 
432  /*
433  * All segments backing this area are pinned, so that DSA can explicitly
434  * control their lifetime (otherwise a newly created segment belonging to
435  * this area might be freed when the only backend that happens to have it
436  * mapped in ends, corrupting the area).
437  */
438  dsm_pin_segment(segment);
439 
440  /* Create a new DSA area with the control object in this segment. */
441  area = create_internal(dsm_segment_address(segment),
442  init_segment_size,
443  tranche_id,
444  dsm_segment_handle(segment), segment,
445  init_segment_size, max_segment_size);
446 
447  /* Clean up when the control segment detaches. */
450 
451  return area;
452 }
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment, size_t init_segment_size, size_t max_segment_size)
Definition: dsa.c:1218
dsm_segment * dsm_create(Size size, int flags)
Definition: dsm.c:516
void dsm_pin_segment(dsm_segment *seg)
Definition: dsm.c:955

References create_internal(), dsa_on_dsm_detach_release_in_place(), dsm_create(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), on_dsm_detach(), and PointerGetDatum().

Referenced by TidStoreCreateShared().

◆ dsa_create_in_place_ext()

dsa_area* dsa_create_in_place_ext ( void *  place,
size_t  size,
int  tranche_id,
dsm_segment segment,
size_t  init_segment_size,
size_t  max_segment_size 
)

Definition at line 471 of file dsa.c.

474 {
475  dsa_area *area;
476 
477  area = create_internal(place, size, tranche_id,
478  DSM_HANDLE_INVALID, NULL,
479  init_segment_size, max_segment_size);
480 
481  /*
482  * Clean up when the control segment detaches, if a containing DSM segment
483  * was provided.
484  */
485  if (segment != NULL)
487  PointerGetDatum(place));
488 
489  return area;
490 }

References create_internal(), dsa_on_dsm_detach_release_in_place(), DSM_HANDLE_INVALID, on_dsm_detach(), PointerGetDatum(), and size.

◆ dsa_detach()

void dsa_detach ( dsa_area area)

Definition at line 1952 of file dsa.c.

1953 {
1954  int i;
1955 
1956  /* Detach from all segments. */
1957  for (i = 0; i <= area->high_segment_index; ++i)
1958  if (area->segment_maps[i].segment != NULL)
1959  dsm_detach(area->segment_maps[i].segment);
1960 
1961  /*
1962  * Note that 'detaching' (= detaching from DSM segments) doesn't include
1963  * 'releasing' (= adjusting the reference count). It would be nice to
1964  * combine these operations, but client code might never get around to
1965  * calling dsa_detach because of an error path, and a detach hook on any
1966  * particular segment is too late to detach other segments in the area
1967  * without risking a 'leak' warning in the non-error path.
1968  */
1969 
1970  /* Free the backend-local area object. */
1971  pfree(area);
1972 }
void pfree(void *pointer)
Definition: mcxt.c:1520

References dsm_detach(), dsa_area::high_segment_index, i, pfree(), dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by DetachSession(), ExecParallelCleanup(), ParallelQueryMain(), pgstat_detach_shmem(), StatsShmemInit(), test_basic(), test_dsa_basic(), test_dsa_resowners(), test_empty(), test_random(), TidStoreDestroy(), and TidStoreDetach().

◆ dsa_dump()

void dsa_dump ( dsa_area area)

Definition at line 1088 of file dsa.c.

1089 {
1090  size_t i,
1091  j;
1092 
1093  /*
1094  * Note: This gives an inconsistent snapshot as it acquires and releases
1095  * individual locks as it goes...
1096  */
1097 
1100  fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
1101  fprintf(stderr, " max_total_segment_size: %zu\n",
1103  fprintf(stderr, " total_segment_size: %zu\n",
1104  area->control->total_segment_size);
1105  fprintf(stderr, " refcnt: %d\n", area->control->refcnt);
1106  fprintf(stderr, " pinned: %c\n", area->control->pinned ? 't' : 'f');
1107  fprintf(stderr, " segment bins:\n");
1108  for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
1109  {
1111  {
1112  dsa_segment_index segment_index;
1113 
1114  if (i == 0)
1115  fprintf(stderr,
1116  " segment bin %zu (no contiguous free pages):\n", i);
1117  else
1118  fprintf(stderr,
1119  " segment bin %zu (at least %d contiguous pages free):\n",
1120  i, 1 << (i - 1));
1121  segment_index = area->control->segment_bins[i];
1122  while (segment_index != DSA_SEGMENT_INDEX_NONE)
1123  {
1124  dsa_segment_map *segment_map;
1125 
1126  segment_map =
1127  get_segment_by_index(area, segment_index);
1128 
1129  fprintf(stderr,
1130  " segment index %zu, usable_pages = %zu, "
1131  "contiguous_pages = %zu, mapped at %p\n",
1132  segment_index,
1133  segment_map->header->usable_pages,
1134  fpm_largest(segment_map->fpm),
1135  segment_map->mapped_address);
1136  segment_index = segment_map->header->next;
1137  }
1138  }
1139  }
1141 
1142  fprintf(stderr, " pools:\n");
1143  for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
1144  {
1145  bool found = false;
1146 
1148  for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
1149  if (DsaPointerIsValid(area->control->pools[i].spans[j]))
1150  found = true;
1151  if (found)
1152  {
1154  fprintf(stderr, " pool for blocks of span objects:\n");
1155  else if (i == DSA_SCLASS_SPAN_LARGE)
1156  fprintf(stderr, " pool for large object spans:\n");
1157  else
1158  fprintf(stderr,
1159  " pool for size class %zu (object size %hu bytes):\n",
1160  i, dsa_size_classes[i]);
1161  for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
1162  {
1163  if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
1164  fprintf(stderr, " fullness class %zu is empty\n", j);
1165  else
1166  {
1167  dsa_pointer span_pointer = area->control->pools[i].spans[j];
1168 
1169  fprintf(stderr, " fullness class %zu:\n", j);
1170  while (DsaPointerIsValid(span_pointer))
1171  {
1172  dsa_area_span *span;
1173 
1174  span = dsa_get_address(area, span_pointer);
1175  fprintf(stderr,
1176  " span descriptor at "
1177  DSA_POINTER_FORMAT ", superblock at "
1179  ", pages = %zu, objects free = %hu/%hu\n",
1180  span_pointer, span->start, span->npages,
1181  span->nallocatable, span->nmax);
1182  span_pointer = span->nextspan;
1183  }
1184  }
1185  }
1186  }
1188  }
1189 }
#define DSA_POINTER_FORMAT
Definition: dsa.h:69
int j
Definition: isn.c:74
#define fprintf
Definition: port.h:242
bool pinned
Definition: dsa.c:312
uint16 nmax
Definition: dsa.c:194

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_POINTER_FORMAT, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DSA_SEGMENT_INDEX_NONE, dsa_size_classes, DsaPointerIsValid, dsa_segment_map::fpm, fpm_largest, fprintf, get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, i, j, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_control::max_total_segment_size, dsa_area_span::nallocatable, dsa_segment_header::next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_control::pinned, dsa_area_control::pools, dsa_area_control::refcnt, dsa_area_control::segment_bins, dsa_area_pool::spans, dsa_area_span::start, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

◆ dsa_free()

void dsa_free ( dsa_area area,
dsa_pointer  dp 
)

Definition at line 826 of file dsa.c.

827 {
828  dsa_segment_map *segment_map;
829  int pageno;
830  dsa_pointer span_pointer;
831  dsa_area_span *span;
832  char *superblock;
833  char *object;
834  size_t size;
835  int size_class;
836 
837  /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
839 
840  /* Locate the object, span and pool. */
841  segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
842  pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
843  span_pointer = segment_map->pagemap[pageno];
844  span = dsa_get_address(area, span_pointer);
845  superblock = dsa_get_address(area, span->start);
846  object = dsa_get_address(area, dp);
847  size_class = span->size_class;
848  size = dsa_size_classes[size_class];
849 
850  /*
851  * Special case for large objects that live in a special span: we return
852  * those pages directly to the free page manager and free the span.
853  */
854  if (span->size_class == DSA_SCLASS_SPAN_LARGE)
855  {
856 
857 #ifdef CLOBBER_FREED_MEMORY
858  memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
859 #endif
860 
861  /* Give pages back to free page manager. */
863  FreePageManagerPut(segment_map->fpm,
865  span->npages);
866 
867  /* Move segment to appropriate bin if necessary. */
868  rebin_segment(area, segment_map);
870 
871  /* Unlink span. */
873  LW_EXCLUSIVE);
874  unlink_span(area, span);
876  /* Free the span object so it can be reused. */
877  dsa_free(area, span_pointer);
878  return;
879  }
880 
881 #ifdef CLOBBER_FREED_MEMORY
882  memset(object, 0x7f, size);
883 #endif
884 
885  LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
886 
887  /* Put the object on the span's freelist. */
888  Assert(object >= superblock);
889  Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
890  Assert((object - superblock) % size == 0);
891  NextFreeObjectIndex(object) = span->firstfree;
892  span->firstfree = (object - superblock) / size;
893  ++span->nallocatable;
894 
895  /*
896  * See if the span needs to moved to a different fullness class, or be
897  * freed so its pages can be given back to the segment.
898  */
899  if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
900  {
901  /*
902  * The block was completely full and is located in the
903  * highest-numbered fullness class, which is never scanned for free
904  * chunks. We must move it to the next-lower fullness class.
905  */
906  unlink_span(area, span);
907  add_span_to_fullness_class(area, span, span_pointer,
909 
910  /*
911  * If this is the only span, and there is no active span, then we
912  * should probably move this span to fullness class 1. (Otherwise if
913  * you allocate exactly all the objects in the only span, it moves to
914  * class 3, then you free them all, it moves to 2, and then is given
915  * back, leaving no active span).
916  */
917  }
918  else if (span->nallocatable == span->nmax &&
919  (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
920  {
921  /*
922  * This entire block is free, and it's not the active block for this
923  * size class. Return the memory to the free page manager. We don't
924  * do this for the active block to prevent hysteresis: if we
925  * repeatedly allocate and free the only chunk in the active block, it
926  * will be very inefficient if we deallocate and reallocate the block
927  * every time.
928  */
929  destroy_superblock(area, span_pointer);
930  }
931 
932  LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
933 }
static void check_for_freed_segments(dsa_area *area)
Definition: dsa.c:2252
static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
Definition: dsa.c:1929
#define DSA_SUPERBLOCK_SIZE
Definition: dsa.c:376
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
Definition: dsa.c:1837

References add_span_to_fullness_class(), Assert, check_for_freed_segments(), destroy_superblock(), DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SUPERBLOCK_SIZE, dsa_area_span::fclass, dsa_area_span::firstfree, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerPut(), get_segment_by_index(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::nmax, dsa_area_span::npages, dsa_segment_map::pagemap, dsa_area_span::prevspan, rebin_segment(), size, dsa_area_span::size_class, dsa_area_span::start, and unlink_span().

Referenced by delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dshash_create(), dshash_destroy(), ExecHashTableDetach(), ExecHashTableDetachBatch(), ExecParallelCleanup(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashRepartitionFirst(), ExecParallelReinitialize(), find_or_make_matching_shared_tupledesc(), pagetable_free(), pgstat_free_entry(), resize(), RT_FREE(), RT_FREE_LEAF(), RT_FREE_NODE(), tbm_free_shared_area(), test_dsa_basic(), and test_dsa_resowners().

◆ dsa_get_address()

void* dsa_get_address ( dsa_area area,
dsa_pointer  dp 
)

Definition at line 942 of file dsa.c.

943 {
945  size_t offset;
946 
947  /* Convert InvalidDsaPointer to NULL. */
948  if (!DsaPointerIsValid(dp))
949  return NULL;
950 
951  /* Process any requests to detach from freed segments. */
953 
954  /* Break the dsa_pointer into its components. */
956  offset = DSA_EXTRACT_OFFSET(dp);
958 
959  /* Check if we need to cause this segment to be mapped in. */
960  if (unlikely(area->segment_maps[index].mapped_address == NULL))
961  {
962  /* Call for effect (we don't need the result). */
964  }
965 
966  return area->segment_maps[index].mapped_address + offset;
967 }

References Assert, check_for_freed_segments(), DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_MAX_SEGMENTS, DsaPointerIsValid, get_segment_by_index(), dsa_segment_map::mapped_address, dsa_area::segment_maps, and unlikely.

Referenced by add_span_to_fullness_class(), alloc_object(), delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_dump(), dsa_free(), dsa_trim(), dshash_attach(), dshash_create(), dshash_destroy(), dshash_dump(), dshash_seq_next(), ensure_active_superblock(), ensure_valid_bucket_pointers(), ExecHashTableDetachBatch(), ExecParallelHashEnsureBatchAccessors(), ExecParallelHashFirstTuple(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashJoinSetUpBatches(), ExecParallelHashNextTuple(), ExecParallelHashPopChunkQueue(), ExecParallelHashRepartitionRest(), ExecParallelHashTableAlloc(), ExecParallelHashTableSetCurrentBatch(), ExecParallelHashTupleAlloc(), find_in_bucket(), find_or_make_matching_shared_tupledesc(), init_span(), insert_into_bucket(), insert_item_into_bucket(), lookup_rowtype_tupdesc_internal(), pagetable_allocate(), ParallelQueryMain(), pgstat_build_snapshot(), pgstat_get_entry_ref(), pgstat_init_entry(), pgstat_reinit_entry(), pgstat_reset_matching_entries(), pgstat_write_statsfile(), resize(), RT_PTR_SET_LOCAL(), SerializeParamExecParams(), share_tupledesc(), shared_record_table_compare(), shared_record_table_hash(), tbm_attach_shared_iterate(), tbm_free_shared_area(), tbm_prepare_shared_iterate(), test_dsa_basic(), test_dsa_resowners(), transfer_first_span(), and unlink_span().

◆ dsa_get_handle()

dsa_handle dsa_get_handle ( dsa_area area)

◆ dsa_get_total_size()

size_t dsa_get_total_size ( dsa_area area)

Definition at line 1027 of file dsa.c.

1028 {
1029  size_t size;
1030 
1032  size = area->control->total_segment_size;
1034 
1035  return size;
1036 }

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), size, and dsa_area_control::total_segment_size.

Referenced by RT_MEMORY_USAGE().

◆ dsa_minimum_size()

size_t dsa_minimum_size ( void  )

Definition at line 1196 of file dsa.c.

1197 {
1198  size_t size;
1199  int pages = 0;
1200 
1201  size = MAXALIGN(sizeof(dsa_area_control)) +
1202  MAXALIGN(sizeof(FreePageManager));
1203 
1204  /* Figure out how many pages we need, including the page map... */
1205  while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
1206  {
1207  ++pages;
1208  size += sizeof(dsa_pointer);
1209  }
1210 
1211  return pages * FPM_PAGE_SIZE;
1212 }

References FPM_PAGE_SIZE, MAXALIGN, and size.

Referenced by create_internal(), ExecInitParallelPlan(), and pgstat_dsa_init_size().

◆ dsa_on_dsm_detach_release_in_place()

void dsa_on_dsm_detach_release_in_place ( dsm_segment segment,
Datum  place 
)

Definition at line 576 of file dsa.c.

577 {
579 }
void dsa_release_in_place(void *place)
Definition: dsa.c:605
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:312

References DatumGetPointer(), and dsa_release_in_place().

Referenced by dsa_attach(), dsa_attach_in_place(), dsa_create_ext(), and dsa_create_in_place_ext().

◆ dsa_on_shmem_exit_release_in_place()

void dsa_on_shmem_exit_release_in_place ( int  code,
Datum  place 
)

Definition at line 590 of file dsa.c.

591 {
593 }

References DatumGetPointer(), and dsa_release_in_place().

◆ dsa_pin()

void dsa_pin ( dsa_area area)

Definition at line 975 of file dsa.c.

976 {
978  if (area->control->pinned)
979  {
981  elog(ERROR, "dsa_area already pinned");
982  }
983  area->control->pinned = true;
984  ++area->control->refcnt;
986 }

References dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

Referenced by init_dsm_registry(), logicalrep_launcher_attach_dshmem(), and StatsShmemInit().

◆ dsa_pin_mapping()

void dsa_pin_mapping ( dsa_area area)

Definition at line 635 of file dsa.c.

636 {
637  int i;
638 
639  if (area->resowner != NULL)
640  {
641  area->resowner = NULL;
642 
643  for (i = 0; i <= area->high_segment_index; ++i)
644  if (area->segment_maps[i].segment != NULL)
646  }
647 }
void dsm_pin_mapping(dsm_segment *seg)
Definition: dsm.c:915

References dsm_pin_mapping(), dsa_area::high_segment_index, i, dsa_area::resowner, dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AttachSession(), GetSessionDsmHandle(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), pgstat_attach_shmem(), and test_create().

◆ dsa_release_in_place()

void dsa_release_in_place ( void *  place)

Definition at line 605 of file dsa.c.

606 {
607  dsa_area_control *control = (dsa_area_control *) place;
608  int i;
609 
610  LWLockAcquire(&control->lock, LW_EXCLUSIVE);
611  Assert(control->segment_header.magic ==
612  (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
613  Assert(control->refcnt > 0);
614  if (--control->refcnt == 0)
615  {
616  for (i = 0; i <= control->high_segment_index; ++i)
617  {
618  dsm_handle handle;
619 
620  handle = control->segment_handles[i];
621  if (handle != DSM_HANDLE_INVALID)
622  dsm_unpin_segment(handle);
623  }
624  }
625  LWLockRelease(&control->lock);
626 }
uint32 dsm_handle
Definition: dsm_impl.h:55
dsa_segment_index high_segment_index
Definition: dsa.c:308

References Assert, DSA_SEGMENT_HEADER_MAGIC, DSM_HANDLE_INVALID, dsm_unpin_segment(), dsa_area_control::handle, dsa_area_control::high_segment_index, i, dsa_area_control::lock, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_area_control::refcnt, dsa_area_control::segment_handles, and dsa_area_control::segment_header.

Referenced by dsa_on_dsm_detach_release_in_place(), and dsa_on_shmem_exit_release_in_place().

◆ dsa_set_size_limit()

void dsa_set_size_limit ( dsa_area area,
size_t  limit 
)

◆ dsa_trim()

void dsa_trim ( dsa_area area)

Definition at line 1043 of file dsa.c.

1044 {
1045  int size_class;
1046 
1047  /*
1048  * Trim in reverse pool order so we get to the spans-of-spans last, just
1049  * in case any become entirely free while processing all the other pools.
1050  */
1051  for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
1052  {
1053  dsa_area_pool *pool = &area->control->pools[size_class];
1054  dsa_pointer span_pointer;
1055 
1056  if (size_class == DSA_SCLASS_SPAN_LARGE)
1057  {
1058  /* Large object frees give back segments aggressively already. */
1059  continue;
1060  }
1061 
1062  /*
1063  * Search fullness class 1 only. That is where we expect to find an
1064  * entirely empty superblock (entirely empty superblocks in other
1065  * fullness classes are returned to the free page map by dsa_free).
1066  */
1067  LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
1068  span_pointer = pool->spans[1];
1069  while (DsaPointerIsValid(span_pointer))
1070  {
1071  dsa_area_span *span = dsa_get_address(area, span_pointer);
1072  dsa_pointer next = span->nextspan;
1073 
1074  if (span->nallocatable == span->nmax)
1075  destroy_superblock(area, span_pointer);
1076 
1077  span_pointer = next;
1078  }
1079  LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
1080  }
1081 }
static int32 next
Definition: blutils.c:221

References dsa_area::control, destroy_superblock(), dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DsaPointerIsValid, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_control::pools, and dsa_area_pool::spans.

◆ dsa_unpin()

void dsa_unpin ( dsa_area area)

Definition at line 994 of file dsa.c.

995 {
997  Assert(area->control->refcnt > 1);
998  if (!area->control->pinned)
999  {
1001  elog(ERROR, "dsa_area not pinned");
1002  }
1003  area->control->pinned = false;
1004  --area->control->refcnt;
1006 }

References Assert, dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

◆ ensure_active_superblock()

static bool ensure_active_superblock ( dsa_area area,
dsa_area_pool pool,
int  size_class 
)
static

Definition at line 1560 of file dsa.c.

1562 {
1563  dsa_pointer span_pointer;
1564  dsa_pointer start_pointer;
1565  size_t obsize = dsa_size_classes[size_class];
1566  size_t nmax;
1567  int fclass;
1568  size_t npages = 1;
1569  size_t first_page;
1570  size_t i;
1571  dsa_segment_map *segment_map;
1572 
1573  Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1574 
1575  /*
1576  * Compute the number of objects that will fit in a block of this size
1577  * class. Span-of-spans blocks are just a single page, and the first
1578  * object isn't available for use because it describes the block-of-spans
1579  * itself.
1580  */
1581  if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1582  nmax = FPM_PAGE_SIZE / obsize - 1;
1583  else
1584  nmax = DSA_SUPERBLOCK_SIZE / obsize;
1585 
1586  /*
1587  * If fullness class 1 is empty, try to find a span to put in it by
1588  * scanning higher-numbered fullness classes (excluding the last one,
1589  * whose blocks are certain to all be completely full).
1590  */
1591  for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
1592  {
1593  span_pointer = pool->spans[fclass];
1594 
1595  while (DsaPointerIsValid(span_pointer))
1596  {
1597  int tfclass;
1598  dsa_area_span *span;
1599  dsa_area_span *nextspan;
1600  dsa_area_span *prevspan;
1601  dsa_pointer next_span_pointer;
1602 
1603  span = (dsa_area_span *)
1604  dsa_get_address(area, span_pointer);
1605  next_span_pointer = span->nextspan;
1606 
1607  /* Figure out what fullness class should contain this span. */
1608  tfclass = (nmax - span->nallocatable)
1609  * (DSA_FULLNESS_CLASSES - 1) / nmax;
1610 
1611  /* Look up next span. */
1612  if (DsaPointerIsValid(span->nextspan))
1613  nextspan = (dsa_area_span *)
1614  dsa_get_address(area, span->nextspan);
1615  else
1616  nextspan = NULL;
1617 
1618  /*
1619  * If utilization has dropped enough that this now belongs in some
1620  * other fullness class, move it there.
1621  */
1622  if (tfclass < fclass)
1623  {
1624  /* Remove from the current fullness class list. */
1625  if (pool->spans[fclass] == span_pointer)
1626  {
1627  /* It was the head; remove it. */
1629  pool->spans[fclass] = span->nextspan;
1630  if (nextspan != NULL)
1631  nextspan->prevspan = InvalidDsaPointer;
1632  }
1633  else
1634  {
1635  /* It was not the head. */
1637  prevspan = (dsa_area_span *)
1638  dsa_get_address(area, span->prevspan);
1639  prevspan->nextspan = span->nextspan;
1640  }
1641  if (nextspan != NULL)
1642  nextspan->prevspan = span->prevspan;
1643 
1644  /* Push onto the head of the new fullness class list. */
1645  span->nextspan = pool->spans[tfclass];
1646  pool->spans[tfclass] = span_pointer;
1647  span->prevspan = InvalidDsaPointer;
1648  if (DsaPointerIsValid(span->nextspan))
1649  {
1650  nextspan = (dsa_area_span *)
1651  dsa_get_address(area, span->nextspan);
1652  nextspan->prevspan = span_pointer;
1653  }
1654  span->fclass = tfclass;
1655  }
1656 
1657  /* Advance to next span on list. */
1658  span_pointer = next_span_pointer;
1659  }
1660 
1661  /* Stop now if we found a suitable block. */
1662  if (DsaPointerIsValid(pool->spans[1]))
1663  return true;
1664  }
1665 
1666  /*
1667  * If there are no blocks that properly belong in fullness class 1, pick
1668  * one from some other fullness class and move it there anyway, so that we
1669  * have an allocation target. Our last choice is to transfer a block
1670  * that's almost empty (and might become completely empty soon if left
1671  * alone), but even that is better than failing, which is what we must do
1672  * if there are no blocks at all with freespace.
1673  */
1674  Assert(!DsaPointerIsValid(pool->spans[1]));
1675  for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
1676  if (transfer_first_span(area, pool, fclass, 1))
1677  return true;
1678  if (!DsaPointerIsValid(pool->spans[1]) &&
1679  transfer_first_span(area, pool, 0, 1))
1680  return true;
1681 
1682  /*
1683  * We failed to find an existing span with free objects, so we need to
1684  * allocate a new superblock and construct a new span to manage it.
1685  *
1686  * First, get a dsa_area_span object to describe the new superblock block
1687  * ... unless this allocation is for a dsa_area_span object, in which case
1688  * that's surely not going to work. We handle that case by storing the
1689  * span describing a block-of-spans inline.
1690  */
1691  if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
1692  {
1693  span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
1694  if (!DsaPointerIsValid(span_pointer))
1695  return false;
1696  npages = DSA_PAGES_PER_SUPERBLOCK;
1697  }
1698 
1699  /* Find or create a segment and allocate the superblock. */
1701  segment_map = get_best_segment(area, npages);
1702  if (segment_map == NULL)
1703  {
1704  segment_map = make_new_segment(area, npages);
1705  if (segment_map == NULL)
1706  {
1708  return false;
1709  }
1710  }
1711 
1712  /*
1713  * This shouldn't happen: get_best_segment() or make_new_segment()
1714  * promised that we can successfully allocate npages.
1715  */
1716  if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
1717  elog(FATAL,
1718  "dsa_allocate could not find %zu free pages for superblock",
1719  npages);
1721 
1722  /* Compute the start of the superblock. */
1723  start_pointer =
1724  DSA_MAKE_POINTER(get_segment_index(area, segment_map),
1725  first_page * FPM_PAGE_SIZE);
1726 
1727  /*
1728  * If this is a block-of-spans, carve the descriptor right out of the
1729  * allocated space.
1730  */
1731  if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1732  {
1733  /*
1734  * We have a pointer into the segment. We need to build a dsa_pointer
1735  * from the segment index and offset into the segment.
1736  */
1737  span_pointer = start_pointer;
1738  }
1739 
1740  /* Initialize span and pagemap. */
1741  init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1742  for (i = 0; i < npages; ++i)
1743  segment_map->pagemap[first_page + i] = span_pointer;
1744 
1745  return true;
1746 }
#define DSA_PAGES_PER_SUPERBLOCK
Definition: dsa.c:82

References alloc_object(), Assert, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_MAKE_POINTER, DSA_PAGES_PER_SUPERBLOCK, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SUPERBLOCK_SIZE, DsaPointerIsValid, elog, FATAL, dsa_area_span::fclass, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerGet(), get_best_segment(), get_segment_index, i, init_span(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), make_new_segment(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_segment_map::pagemap, dsa_area_span::prevspan, dsa_area_pool::spans, and transfer_first_span().

Referenced by alloc_object().

◆ get_best_segment()

static dsa_segment_map * get_best_segment ( dsa_area area,
size_t  npages 
)
static

Definition at line 2010 of file dsa.c.

2011 {
2012  size_t bin;
2013 
2016 
2017  /*
2018  * Start searching from the first bin that *might* have enough contiguous
2019  * pages.
2020  */
2021  for (bin = contiguous_pages_to_segment_bin(npages);
2022  bin < DSA_NUM_SEGMENT_BINS;
2023  ++bin)
2024  {
2025  /*
2026  * The minimum contiguous size that any segment in this bin should
2027  * have. We'll re-bin if we see segments with fewer.
2028  */
2029  size_t threshold = (size_t) 1 << (bin - 1);
2030  dsa_segment_index segment_index;
2031 
2032  /* Search this bin for a segment with enough contiguous space. */
2033  segment_index = area->control->segment_bins[bin];
2034  while (segment_index != DSA_SEGMENT_INDEX_NONE)
2035  {
2036  dsa_segment_map *segment_map;
2037  dsa_segment_index next_segment_index;
2038  size_t contiguous_pages;
2039 
2040  segment_map = get_segment_by_index(area, segment_index);
2041  next_segment_index = segment_map->header->next;
2042  contiguous_pages = fpm_largest(segment_map->fpm);
2043 
2044  /* Not enough for the request, still enough for this bin. */
2045  if (contiguous_pages >= threshold && contiguous_pages < npages)
2046  {
2047  segment_index = next_segment_index;
2048  continue;
2049  }
2050 
2051  /* Re-bin it if it's no longer in the appropriate bin. */
2052  if (contiguous_pages < threshold)
2053  {
2054  rebin_segment(area, segment_map);
2055 
2056  /*
2057  * But fall through to see if it's enough to satisfy this
2058  * request anyway....
2059  */
2060  }
2061 
2062  /* Check if we are done. */
2063  if (contiguous_pages >= npages)
2064  return segment_map;
2065 
2066  /* Continue searching the same bin. */
2067  segment_index = next_segment_index;
2068  }
2069  }
2070 
2071  /* Not found. */
2072  return NULL;
2073 }

References Assert, check_for_freed_segments_locked(), contiguous_pages_to_segment_bin(), dsa_area::control, DSA_AREA_LOCK, DSA_NUM_SEGMENT_BINS, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), dsa_segment_map::header, LWLockHeldByMe(), dsa_segment_header::next, rebin_segment(), and dsa_area_control::segment_bins.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ get_segment_by_index()

static dsa_segment_map * get_segment_by_index ( dsa_area area,
dsa_segment_index  index 
)
static

Definition at line 1757 of file dsa.c.

1758 {
1759  if (unlikely(area->segment_maps[index].mapped_address == NULL))
1760  {
1761  dsm_handle handle;
1762  dsm_segment *segment;
1763  dsa_segment_map *segment_map;
1764  ResourceOwner oldowner;
1765 
1766  /*
1767  * If we are reached by dsa_free or dsa_get_address, there must be at
1768  * least one object allocated in the referenced segment. Otherwise,
1769  * their caller has a double-free or access-after-free bug, which we
1770  * have no hope of detecting. So we know it's safe to access this
1771  * array slot without holding a lock; it won't change underneath us.
1772  * Furthermore, we know that we can see the latest contents of the
1773  * slot, as explained in check_for_freed_segments, which those
1774  * functions call before arriving here.
1775  */
1776  handle = area->control->segment_handles[index];
1777 
1778  /* It's an error to try to access an unused slot. */
1779  if (handle == DSM_HANDLE_INVALID)
1780  elog(ERROR,
1781  "dsa_area could not attach to a segment that has been freed");
1782 
1783  oldowner = CurrentResourceOwner;
1785  segment = dsm_attach(handle);
1786  CurrentResourceOwner = oldowner;
1787  if (segment == NULL)
1788  elog(ERROR, "dsa_area could not attach to segment");
1789  segment_map = &area->segment_maps[index];
1790  segment_map->segment = segment;
1791  segment_map->mapped_address = dsm_segment_address(segment);
1792  segment_map->header =
1793  (dsa_segment_header *) segment_map->mapped_address;
1794  segment_map->fpm = (FreePageManager *)
1795  (segment_map->mapped_address +
1796  MAXALIGN(sizeof(dsa_segment_header)));
1797  segment_map->pagemap = (dsa_pointer *)
1798  (segment_map->mapped_address +
1799  MAXALIGN(sizeof(dsa_segment_header)) +
1800  MAXALIGN(sizeof(FreePageManager)));
1801 
1802  /* Remember the highest index this backend has ever mapped. */
1803  if (area->high_segment_index < index)
1804  area->high_segment_index = index;
1805 
1806  Assert(segment_map->header->magic ==
1808  }
1809 
1810  /*
1811  * Callers of dsa_get_address() and dsa_free() don't hold the area lock,
1812  * but it's a bug in the calling code and undefined behavior if the
1813  * address is not live (ie if the segment might possibly have been freed,
1814  * they're trying to use a dangling pointer).
1815  *
1816  * For dsa.c code that holds the area lock to manipulate segment_bins
1817  * lists, it would be a bug if we ever reach a freed segment here. After
1818  * it's marked as freed, the only thing any backend should do with it is
1819  * unmap it, and it should always have done that in
1820  * check_for_freed_segments_locked() before arriving here to resolve an
1821  * index to a segment_map.
1822  *
1823  * Either way we can assert that we aren't returning a freed segment.
1824  */
1825  Assert(!area->segment_maps[index].header->freed);
1826 
1827  return &area->segment_maps[index];
1828 }

References Assert, dsa_area::control, CurrentResourceOwner, DSA_SEGMENT_HEADER_MAGIC, dsm_attach(), DSM_HANDLE_INVALID, dsm_segment_address(), elog, ERROR, dsa_segment_map::fpm, dsa_segment_header::freed, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, dsa_segment_header::magic, dsa_segment_map::mapped_address, MAXALIGN, dsa_segment_map::pagemap, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area::segment_maps, and unlikely.

Referenced by destroy_superblock(), dsa_dump(), dsa_free(), dsa_get_address(), get_best_segment(), make_new_segment(), rebin_segment(), and unlink_segment().

◆ init_span()

static void init_span ( dsa_area area,
dsa_pointer  span_pointer,
dsa_area_pool pool,
dsa_pointer  start,
size_t  npages,
uint16  size_class 
)
static

Definition at line 1377 of file dsa.c.

1381 {
1382  dsa_area_span *span = dsa_get_address(area, span_pointer);
1383  size_t obsize = dsa_size_classes[size_class];
1384 
1385  /*
1386  * The per-pool lock must be held because we manipulate the span list for
1387  * this pool.
1388  */
1389  Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1390 
1391  /* Push this span onto the front of the span list for fullness class 1. */
1392  if (DsaPointerIsValid(pool->spans[1]))
1393  {
1394  dsa_area_span *head = (dsa_area_span *)
1395  dsa_get_address(area, pool->spans[1]);
1396 
1397  head->prevspan = span_pointer;
1398  }
1399  span->pool = DsaAreaPoolToDsaPointer(area, pool);
1400  span->nextspan = pool->spans[1];
1401  span->prevspan = InvalidDsaPointer;
1402  pool->spans[1] = span_pointer;
1403 
1404  span->start = start;
1405  span->npages = npages;
1406  span->size_class = size_class;
1407  span->ninitialized = 0;
1408  if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1409  {
1410  /*
1411  * A block-of-spans contains its own descriptor, so mark one object as
1412  * initialized and reduce the count of allocatable objects by one.
1413  * Doing this here has the side effect of also reducing nmax by one,
1414  * which is important to make sure we free this object at the correct
1415  * time.
1416  */
1417  span->ninitialized = 1;
1418  span->nallocatable = FPM_PAGE_SIZE / obsize - 1;
1419  }
1420  else if (size_class != DSA_SCLASS_SPAN_LARGE)
1421  span->nallocatable = DSA_SUPERBLOCK_SIZE / obsize;
1423  span->nmax = span->nallocatable;
1424  span->fclass = 1;
1425 }
#define DsaAreaPoolToDsaPointer(area, p)
Definition: dsa.c:322
return str start

References Assert, dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DSA_SUPERBLOCK_SIZE, DsaAreaPoolToDsaPointer, DsaPointerIsValid, dsa_area_span::fclass, dsa_area_span::firstfree, FPM_PAGE_SIZE, InvalidDsaPointer, LWLockHeldByMe(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_area_span::ninitialized, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_span::pool, dsa_area_span::prevspan, dsa_area_span::size_class, dsa_area_pool::spans, dsa_area_span::start, and start.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ make_new_segment()

static dsa_segment_map * make_new_segment ( dsa_area area,
size_t  requested_pages 
)
static

Definition at line 2081 of file dsa.c.

2082 {
2083  dsa_segment_index new_index;
2084  size_t metadata_bytes;
2085  size_t total_size;
2086  size_t total_pages;
2087  size_t usable_pages;
2088  dsa_segment_map *segment_map;
2089  dsm_segment *segment;
2090  ResourceOwner oldowner;
2091 
2093 
2094  /* Find a segment slot that is not in use (linearly for now). */
2095  for (new_index = 1; new_index < DSA_MAX_SEGMENTS; ++new_index)
2096  {
2097  if (area->control->segment_handles[new_index] == DSM_HANDLE_INVALID)
2098  break;
2099  }
2100  if (new_index == DSA_MAX_SEGMENTS)
2101  return NULL;
2102 
2103  /*
2104  * If the total size limit is already exceeded, then we exit early and
2105  * avoid arithmetic wraparound in the unsigned expressions below.
2106  */
2107  if (area->control->total_segment_size >=
2109  return NULL;
2110 
2111  /*
2112  * The size should be at least as big as requested, and at least big
2113  * enough to follow a geometric series that approximately doubles the
2114  * total storage each time we create a new segment. We use geometric
2115  * growth because the underlying DSM system isn't designed for large
2116  * numbers of segments (otherwise we might even consider just using one
2117  * DSM segment for each large allocation and for each superblock, and then
2118  * we wouldn't need to use FreePageManager).
2119  *
2120  * We decide on a total segment size first, so that we produce tidy
2121  * power-of-two sized segments. This is a good property to have if we
2122  * move to huge pages in the future. Then we work back to the number of
2123  * pages we can fit.
2124  */
2126  ((size_t) 1 << (new_index / DSA_NUM_SEGMENTS_AT_EACH_SIZE));
2130  area->control->total_segment_size);
2131 
2132  total_pages = total_size / FPM_PAGE_SIZE;
2133  metadata_bytes =
2134  MAXALIGN(sizeof(dsa_segment_header)) +
2135  MAXALIGN(sizeof(FreePageManager)) +
2136  sizeof(dsa_pointer) * total_pages;
2137 
2138  /* Add padding up to next page boundary. */
2139  if (metadata_bytes % FPM_PAGE_SIZE != 0)
2140  metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
2141  if (total_size <= metadata_bytes)
2142  return NULL;
2143  usable_pages = (total_size - metadata_bytes) / FPM_PAGE_SIZE;
2144  Assert(metadata_bytes + usable_pages * FPM_PAGE_SIZE <= total_size);
2145 
2146  /* See if that is enough... */
2147  if (requested_pages > usable_pages)
2148  {
2149  /*
2150  * We'll make an odd-sized segment, working forward from the requested
2151  * number of pages.
2152  */
2153  usable_pages = requested_pages;
2154  metadata_bytes =
2155  MAXALIGN(sizeof(dsa_segment_header)) +
2156  MAXALIGN(sizeof(FreePageManager)) +
2157  usable_pages * sizeof(dsa_pointer);
2158 
2159  /* Add padding up to next page boundary. */
2160  if (metadata_bytes % FPM_PAGE_SIZE != 0)
2161  metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
2162  total_size = metadata_bytes + usable_pages * FPM_PAGE_SIZE;
2163 
2164  /* Is that too large for dsa_pointer's addressing scheme? */
2166  return NULL;
2167 
2168  /* Would that exceed the limit? */
2170  area->control->total_segment_size)
2171  return NULL;
2172  }
2173 
2174  /* Create the segment. */
2175  oldowner = CurrentResourceOwner;
2177  segment = dsm_create(total_size, 0);
2178  CurrentResourceOwner = oldowner;
2179  if (segment == NULL)
2180  return NULL;
2181  dsm_pin_segment(segment);
2182 
2183  /* Store the handle in shared memory to be found by index. */
2184  area->control->segment_handles[new_index] =
2185  dsm_segment_handle(segment);
2186  /* Track the highest segment index in the history of the area. */
2187  if (area->control->high_segment_index < new_index)
2188  area->control->high_segment_index = new_index;
2189  /* Track the highest segment index this backend has ever mapped. */
2190  if (area->high_segment_index < new_index)
2191  area->high_segment_index = new_index;
2192  /* Track total size of all segments. */
2196 
2197  /* Build a segment map for this segment in this backend. */
2198  segment_map = &area->segment_maps[new_index];
2199  segment_map->segment = segment;
2200  segment_map->mapped_address = dsm_segment_address(segment);
2201  segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
2202  segment_map->fpm = (FreePageManager *)
2203  (segment_map->mapped_address +
2204  MAXALIGN(sizeof(dsa_segment_header)));
2205  segment_map->pagemap = (dsa_pointer *)
2206  (segment_map->mapped_address +
2207  MAXALIGN(sizeof(dsa_segment_header)) +
2208  MAXALIGN(sizeof(FreePageManager)));
2209 
2210  /* Set up the free page map. */
2211  FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
2212  FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
2213  usable_pages);
2214 
2215  /* Set up the segment header and put it in the appropriate bin. */
2216  segment_map->header->magic =
2217  DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ new_index;
2218  segment_map->header->usable_pages = usable_pages;
2219  segment_map->header->size = total_size;
2220  segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
2221  segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
2222  segment_map->header->next =
2223  area->control->segment_bins[segment_map->header->bin];
2224  segment_map->header->freed = false;
2225  area->control->segment_bins[segment_map->header->bin] = new_index;
2226  if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2227  {
2229  get_segment_by_index(area, segment_map->header->next);
2230 
2231  Assert(next->header->bin == segment_map->header->bin);
2232  next->header->prev = new_index;
2233  }
2234 
2235  return segment_map;
2236 }
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE
Definition: dsa.c:69
int64 total_size
Definition: pg_checksums.c:63

References Assert, dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, CurrentResourceOwner, DSA_AREA_LOCK, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_NUM_SEGMENTS_AT_EACH_SIZE, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, dsm_create(), DSM_HANDLE_INVALID, dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, FreePageManagerInitialize(), FreePageManagerPut(), get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, dsa_area_control::high_segment_index, dsa_area::high_segment_index, dsa_area_control::init_segment_size, LWLockHeldByMe(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area_control::max_segment_size, dsa_area_control::max_total_segment_size, MAXALIGN, Min, next, dsa_segment_header::next, dsa_segment_map::pagemap, dsa_segment_header::prev, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, total_size, and dsa_segment_header::usable_pages.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ rebin_segment()

static void rebin_segment ( dsa_area area,
dsa_segment_map segment_map 
)
static

Definition at line 2316 of file dsa.c.

2317 {
2318  size_t new_bin;
2319  dsa_segment_index segment_index;
2320 
2321  new_bin = contiguous_pages_to_segment_bin(fpm_largest(segment_map->fpm));
2322  if (segment_map->header->bin == new_bin)
2323  return;
2324 
2325  /* Remove it from its current bin. */
2326  unlink_segment(area, segment_map);
2327 
2328  /* Push it onto the front of its new bin. */
2329  segment_index = get_segment_index(area, segment_map);
2330  segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
2331  segment_map->header->next = area->control->segment_bins[new_bin];
2332  segment_map->header->bin = new_bin;
2333  area->control->segment_bins[new_bin] = segment_index;
2334  if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2335  {
2337 
2338  next = get_segment_by_index(area, segment_map->header->next);
2339  Assert(next->header->bin == new_bin);
2340  next->header->prev = segment_index;
2341  }
2342 }

References Assert, dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), get_segment_index, dsa_segment_map::header, next, dsa_segment_header::next, dsa_segment_header::prev, dsa_area_control::segment_bins, and unlink_segment().

Referenced by destroy_superblock(), dsa_free(), and get_best_segment().

◆ transfer_first_span()

static bool transfer_first_span ( dsa_area area,
dsa_area_pool pool,
int  fromclass,
int  toclass 
)
static

Definition at line 1432 of file dsa.c.

1434 {
1435  dsa_pointer span_pointer;
1436  dsa_area_span *span;
1437  dsa_area_span *nextspan;
1438 
1439  /* Can't do it if source list is empty. */
1440  span_pointer = pool->spans[fromclass];
1441  if (!DsaPointerIsValid(span_pointer))
1442  return false;
1443 
1444  /* Remove span from head of source list. */
1445  span = dsa_get_address(area, span_pointer);
1446  pool->spans[fromclass] = span->nextspan;
1447  if (DsaPointerIsValid(span->nextspan))
1448  {
1449  nextspan = (dsa_area_span *)
1450  dsa_get_address(area, span->nextspan);
1451  nextspan->prevspan = InvalidDsaPointer;
1452  }
1453 
1454  /* Add span to head of target list. */
1455  span->nextspan = pool->spans[toclass];
1456  pool->spans[toclass] = span_pointer;
1457  if (DsaPointerIsValid(span->nextspan))
1458  {
1459  nextspan = (dsa_area_span *)
1460  dsa_get_address(area, span->nextspan);
1461  nextspan->prevspan = span_pointer;
1462  }
1463  span->fclass = toclass;
1464 
1465  return true;
1466 }

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, InvalidDsaPointer, dsa_area_span::nextspan, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by alloc_object(), and ensure_active_superblock().

◆ unlink_segment()

static void unlink_segment ( dsa_area area,
dsa_segment_map segment_map 
)
static

Definition at line 1978 of file dsa.c.

1979 {
1980  if (segment_map->header->prev != DSA_SEGMENT_INDEX_NONE)
1981  {
1982  dsa_segment_map *prev;
1983 
1984  prev = get_segment_by_index(area, segment_map->header->prev);
1985  prev->header->next = segment_map->header->next;
1986  }
1987  else
1988  {
1989  Assert(area->control->segment_bins[segment_map->header->bin] ==
1990  get_segment_index(area, segment_map));
1991  area->control->segment_bins[segment_map->header->bin] =
1992  segment_map->header->next;
1993  }
1994  if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
1995  {
1997 
1998  next = get_segment_by_index(area, segment_map->header->next);
1999  next->header->prev = segment_map->header->prev;
2000  }
2001 }

References Assert, dsa_segment_header::bin, dsa_area::control, DSA_SEGMENT_INDEX_NONE, get_segment_by_index(), get_segment_index, dsa_segment_map::header, next, dsa_segment_header::next, dsa_segment_header::prev, and dsa_area_control::segment_bins.

Referenced by destroy_superblock(), and rebin_segment().

◆ unlink_span()

static void unlink_span ( dsa_area area,
dsa_area_span span 
)
static

Definition at line 1906 of file dsa.c.

1907 {
1908  if (DsaPointerIsValid(span->nextspan))
1909  {
1910  dsa_area_span *next = dsa_get_address(area, span->nextspan);
1911 
1912  next->prevspan = span->prevspan;
1913  }
1914  if (DsaPointerIsValid(span->prevspan))
1915  {
1916  dsa_area_span *prev = dsa_get_address(area, span->prevspan);
1917 
1918  prev->nextspan = span->nextspan;
1919  }
1920  else
1921  {
1922  dsa_area_pool *pool = dsa_get_address(area, span->pool);
1923 
1924  pool->spans[span->fclass] = span->nextspan;
1925  }
1926 }

References dsa_get_address(), DsaPointerIsValid, dsa_area_span::fclass, next, dsa_area_span::nextspan, dsa_area_span::pool, dsa_area_span::prevspan, and dsa_area_pool::spans.

Referenced by destroy_superblock(), and dsa_free().

Variable Documentation

◆ dsa_size_class_map

const uint8 dsa_size_class_map[]
static
Initial value:
= {
2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
}

Definition at line 248 of file dsa.c.

Referenced by dsa_allocate_extended().

◆ dsa_size_classes

const uint16 dsa_size_classes[]
static
Initial value:
= {
sizeof(dsa_area_span), 0,
8, 16, 24, 32, 40, 48, 56, 64,
80, 96, 112, 128,
160, 192, 224, 256,
320, 384, 448, 512,
640, 768, 896, 1024,
1280, 1560, 1816, 2048,
2616, 3120, 3640, 4096,
5456, 6552, 7280, 8192
}

Definition at line 225 of file dsa.c.

Referenced by alloc_object(), dsa_allocate_extended(), dsa_dump(), dsa_free(), ensure_active_superblock(), and init_span().