69 #define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2
75 #define DSA_MAX_SEGMENTS \
76 Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
79 #define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
82 #define DSA_PAGES_PER_SUPERBLOCK 16
89 #define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608
92 #define DSA_MAKE_POINTER(segment_number, offset) \
93 (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
96 #define DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)
99 #define DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)
105 #define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)
111 #define DSA_NUM_SEGMENT_BINS 16
132 #define DSA_AREA_LOCK(area) (&area->control->lock)
133 #define DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)
202 #define NextFreeObjectIndex(object) (* (uint16 *) (object))
227 8, 16, 24, 32, 40, 48, 56, 64,
232 1280, 1560, 1816, 2048,
233 2616, 3120, 3640, 4096,
234 5456, 6552, 7280, 8192
236 #define DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)
239 #define DSA_SCLASS_BLOCK_OF_SPANS 0
240 #define DSA_SCLASS_SPAN_LARGE 1
249 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
250 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
251 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
252 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
253 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
254 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
255 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
256 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
258 #define DSA_SIZE_CLASS_MAP_QUANTUM 8
266 #define DSA_FULLNESS_CLASSES 4
322 #define DsaAreaPoolToDsaPointer(area, p) \
323 DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
375 #define DSA_SPAN_NOTHING_FREE ((uint16) -1)
376 #define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
379 #define get_segment_index(area, segment_map_ptr) \
380 (segment_map_ptr - &area->segment_maps[0])
386 int fromclass,
int toclass);
403 size_t init_segment_size,
404 size_t max_segment_size);
421 dsa_create_ext(
int tranche_id,
size_t init_segment_size,
size_t max_segment_size)
445 init_segment_size, max_segment_size);
473 size_t init_segment_size,
size_t max_segment_size)
479 init_segment_size, max_segment_size);
522 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
523 errmsg(
"could not attach to dynamic shared area")));
614 if (--control->
refcnt == 0)
683 elog(
ERROR,
"invalid DSA memory alloc request size %zu",
size);
706 (
errcode(ERRCODE_OUT_OF_MEMORY),
708 errdetail(
"Failed on DSA request of size %zu.",
717 if (segment_map == NULL)
719 if (segment_map == NULL)
728 (
errcode(ERRCODE_OUT_OF_MEMORY),
730 errdetail(
"Failed on DSA request of size %zu.",
744 "dsa_allocate could not find %zu free pages", npages);
753 init_span(area, span_pointer, pool, start_pointer, npages,
755 segment_map->
pagemap[first_page] = span_pointer;
762 return start_pointer;
786 uint16 mid = (min + max) / 2;
789 if (class_size <
size)
809 (
errcode(ERRCODE_OUT_OF_MEMORY),
843 span_pointer = segment_map->
pagemap[pageno];
857 #ifdef CLOBBER_FREED_MEMORY
881 #ifdef CLOBBER_FREED_MEMORY
882 memset(
object, 0x7f,
size);
888 Assert(
object >= superblock);
1068 span_pointer = pool->
spans[1];
1077 span_pointer =
next;
1101 fprintf(stderr,
" max_total_segment_size: %zu\n",
1103 fprintf(stderr,
" total_segment_size: %zu\n",
1107 fprintf(stderr,
" segment bins:\n");
1116 " segment bin %zu (no contiguous free pages):\n",
i);
1119 " segment bin %zu (at least %d contiguous pages free):\n",
1130 " segment index %zu, usable_pages = %zu, "
1131 "contiguous_pages = %zu, mapped at %p\n",
1154 fprintf(stderr,
" pool for blocks of span objects:\n");
1156 fprintf(stderr,
" pool for large object spans:\n");
1159 " pool for size class %zu (object size %hu bytes):\n",
1164 fprintf(stderr,
" fullness class %zu is empty\n",
j);
1169 fprintf(stderr,
" fullness class %zu:\n",
j);
1176 " span descriptor at "
1179 ", pages = %zu, objects free = %hu/%hu\n",
1222 size_t init_segment_size,
size_t max_segment_size)
1227 size_t usable_pages;
1229 size_t metadata_bytes;
1234 Assert(max_segment_size >= init_segment_size);
1239 elog(
ERROR,
"dsa_area space must be at least %zu, but %zu provided",
1259 memset(place, 0,
sizeof(*control));
1267 control->
handle = control_handle;
1296 segment_map->
segment = control_segment;
1311 if (usable_pages > 0)
1348 segment_map->
segment = segment;
1359 if (control->
refcnt == 0)
1363 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1364 errmsg(
"could not attach to dynamic shared area")));
1402 pool->
spans[1] = span_pointer;
1440 span_pointer = pool->
spans[fromclass];
1456 pool->
spans[toclass] = span_pointer;
1510 block = span->
start;
1593 span_pointer = pool->
spans[fclass];
1605 next_span_pointer = span->
nextspan;
1622 if (tfclass < fclass)
1625 if (pool->
spans[fclass] == span_pointer)
1630 if (nextspan != NULL)
1641 if (nextspan != NULL)
1646 pool->
spans[tfclass] = span_pointer;
1658 span_pointer = next_span_pointer;
1702 if (segment_map == NULL)
1705 if (segment_map == NULL)
1718 "dsa_allocate could not find %zu free pages for superblock",
1737 span_pointer = start_pointer;
1741 init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1742 for (
i = 0;
i < npages; ++
i)
1743 segment_map->
pagemap[first_page +
i] = span_pointer;
1781 "dsa_area could not attach to a segment that has been freed");
1787 if (segment == NULL)
1788 elog(
ERROR,
"dsa_area could not attach to segment");
1790 segment_map->
segment = segment;
1882 segment_map->
header = NULL;
1888 if (segment_map->
header != NULL)
1938 pool->
spans[fclass]);
1944 pool->
spans[fclass] = span_pointer;
2029 size_t threshold = (size_t) 1 << (bin - 1);
2038 size_t contiguous_pages;
2041 next_segment_index = segment_map->
header->
next;
2045 if (contiguous_pages >= threshold && contiguous_pages < npages)
2047 segment_index = next_segment_index;
2052 if (contiguous_pages < threshold)
2063 if (contiguous_pages >= npages)
2067 segment_index = next_segment_index;
2084 size_t metadata_bytes;
2087 size_t usable_pages;
2147 if (requested_pages > usable_pages)
2153 usable_pages = requested_pages;
2179 if (segment == NULL)
2199 segment_map->
segment = segment;
2232 next->header->prev = new_index;
2254 size_t freed_segment_counter;
2290 size_t freed_segment_counter;
2322 if (segment_map->
header->
bin == new_bin)
2340 next->header->prev = segment_index;
#define pg_read_barrier()
#define Assert(condition)
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
static void check_for_freed_segments(dsa_area *area)
static const uint16 dsa_size_classes[]
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
#define DSA_AREA_LOCK(area)
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE
dsa_area * dsa_create_in_place_ext(void *place, size_t size, int tranche_id, dsm_segment *segment, size_t init_segment_size, size_t max_segment_size)
static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)
#define DSA_SEGMENT_INDEX_NONE
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment, size_t init_segment_size, size_t max_segment_size)
#define DSA_SEGMENT_HEADER_MAGIC
void dsa_trim(dsa_area *area)
#define DSA_SPAN_NOTHING_FREE
#define DSA_MAKE_POINTER(segment_number, offset)
#define get_segment_index(area, segment_map_ptr)
void dsa_on_shmem_exit_release_in_place(int code, Datum place)
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
static dsa_pointer alloc_object(dsa_area *area, int size_class)
#define DSA_PAGES_PER_SUPERBLOCK
#define DSA_SIZE_CLASS_MAP_QUANTUM
static size_t contiguous_pages_to_segment_bin(size_t n)
static const uint8 dsa_size_class_map[]
dsa_pointer dsa_allocate_extended(dsa_area *area, size_t size, int flags)
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
size_t dsa_get_total_size(dsa_area *area)
dsa_area * dsa_attach(dsa_handle handle)
#define DSA_SUPERBLOCK_SIZE
#define DsaAreaPoolToDsaPointer(area, p)
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
static void check_for_freed_segments_locked(dsa_area *area)
#define DSA_EXTRACT_OFFSET(dp)
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
static void rebin_segment(dsa_area *area, dsa_segment_map *segment_map)
#define DSA_SCLASS_LOCK(area, sclass)
void dsa_release_in_place(void *place)
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
void dsa_set_size_limit(dsa_area *area, size_t limit)
#define DSA_SCLASS_BLOCK_OF_SPANS
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
static void unlink_span(dsa_area *area, dsa_area_span *span)
#define DSA_SCLASS_SPAN_LARGE
#define DSA_NUM_SIZE_CLASSES
void dsa_unpin(dsa_area *area)
void dsa_pin_mapping(dsa_area *area)
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
#define NextFreeObjectIndex(object)
void dsa_dump(dsa_area *area)
dsa_handle dsa_get_handle(dsa_area *area)
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
void dsa_detach(dsa_area *area)
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
dsa_area * dsa_attach_in_place(void *place, dsm_segment *segment)
#define DSA_FULLNESS_CLASSES
void dsa_free(dsa_area *area, dsa_pointer dp)
#define DSA_NUM_SEGMENT_BINS
dsa_area * dsa_create_ext(int tranche_id, size_t init_segment_size, size_t max_segment_size)
size_t dsa_minimum_size(void)
void dsa_pin(dsa_area *area)
#define DSA_POINTER_FORMAT
#define DSA_MIN_SEGMENT_SIZE
#define InvalidDsaPointer
#define DSA_HANDLE_INVALID
#define DsaPointerIsValid(x)
#define DSA_MAX_SEGMENT_SIZE
dsm_handle dsm_segment_handle(dsm_segment *seg)
void * dsm_segment_address(dsm_segment *seg)
void dsm_detach(dsm_segment *seg)
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
dsm_segment * dsm_attach(dsm_handle h)
dsm_segment * dsm_create(Size size, int flags)
void dsm_pin_mapping(dsm_segment *seg)
void dsm_unpin_segment(dsm_handle handle)
void dsm_pin_segment(dsm_segment *seg)
#define DSM_HANDLE_INVALID
int errdetail(const char *fmt,...)
int errcode(int sqlerrcode)
int errmsg(const char *fmt,...)
#define ereport(elevel,...)
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
#define fpm_size_to_pages(sz)
bool LWLockHeldByMe(LWLock *lock)
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
void LWLockRelease(LWLock *lock)
void LWLockInitialize(LWLock *lock, int tranche_id)
void pfree(void *pointer)
#define AllocHugeSizeIsValid(size)
#define AllocSizeIsValid(size)
#define pg_leftmost_one_pos_size_t
static Datum PointerGetDatum(const void *X)
static Pointer DatumGetPointer(Datum X)
ResourceOwner CurrentResourceOwner
static pg_noinline void Size size
dsa_segment_header segment_header
size_t total_segment_size
dsa_segment_index high_segment_index
size_t max_total_segment_size
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
size_t freed_segment_counter
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
dsa_pointer spans[DSA_FULLNESS_CLASSES]
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
dsa_segment_index high_segment_index
size_t freed_segment_counter
dsa_area_control * control
dsa_segment_header * header