69 #define DSA_INITIAL_SEGMENT_SIZE ((size_t) (1 * 1024 * 1024))
78 #define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2
85 #if SIZEOF_DSA_POINTER == 4
86 #define DSA_OFFSET_WIDTH 27
88 #define DSA_OFFSET_WIDTH 40
95 #define DSA_MAX_SEGMENTS \
96 Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
99 #define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
102 #define DSA_MAX_SEGMENT_SIZE ((size_t) 1 << DSA_OFFSET_WIDTH)
105 #define DSA_PAGES_PER_SUPERBLOCK 16
112 #define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608
115 #define DSA_MAKE_POINTER(segment_number, offset) \
116 (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
119 #define DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)
122 #define DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)
128 #define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)
134 #define DSA_NUM_SEGMENT_BINS 16
155 #define DSA_AREA_LOCK(area) (&area->control->lock)
156 #define DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)
225 #define NextFreeObjectIndex(object) (* (uint16 *) (object))
250 8, 16, 24, 32, 40, 48, 56, 64,
255 1280, 1560, 1816, 2048,
256 2616, 3120, 3640, 4096,
257 5456, 6552, 7280, 8192
259 #define DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)
262 #define DSA_SCLASS_BLOCK_OF_SPANS 0
263 #define DSA_SCLASS_SPAN_LARGE 1
272 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
273 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
274 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
275 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
276 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
277 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
278 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
279 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
281 #define DSA_SIZE_CLASS_MAP_QUANTUM 8
289 #define DSA_FULLNESS_CLASSES 4
341 #define DsaAreaPoolToDsaPointer(area, p) \
342 DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
389 #define DSA_SPAN_NOTHING_FREE ((uint16) -1)
390 #define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
393 #define get_segment_index(area, segment_map_ptr) \
394 (segment_map_ptr - &area->segment_maps[0])
400 int fromclass,
int toclass);
530 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
531 errmsg(
"could not attach to dynamic shared area")));
622 if (--control->
refcnt == 0)
689 elog(
ERROR,
"invalid DSA memory alloc request size %zu", size);
712 (
errcode(ERRCODE_OUT_OF_MEMORY),
714 errdetail(
"Failed on DSA request of size %zu.",
723 if (segment_map == NULL)
725 if (segment_map == NULL)
734 (
errcode(ERRCODE_OUT_OF_MEMORY),
736 errdetail(
"Failed on DSA request of size %zu.",
750 "dsa_allocate could not find %zu free pages", npages);
759 init_span(area, span_pointer, pool, start_pointer, npages,
761 segment_map->
pagemap[first_page] = span_pointer;
768 return start_pointer;
792 uint16 mid = (min + max) / 2;
795 if (class_size < size)
815 (
errcode(ERRCODE_OUT_OF_MEMORY),
817 errdetail(
"Failed on DSA request of size %zu.", size)));
849 span_pointer = segment_map->
pagemap[pageno];
863 #ifdef CLOBBER_FREED_MEMORY
883 #ifdef CLOBBER_FREED_MEMORY
884 memset(
object, 0x7f, size);
890 Assert(
object >= superblock);
892 Assert((
object - superblock) % size == 0);
894 span->
firstfree = (
object - superblock) / size;
1057 span_pointer = pool->
spans[1];
1066 span_pointer =
next;
1090 fprintf(stderr,
" max_total_segment_size: %zu\n",
1092 fprintf(stderr,
" total_segment_size: %zu\n",
1096 fprintf(stderr,
" segment bins:\n");
1104 " segment bin %zu (at least %d contiguous pages free):\n",
1115 " segment index %zu, usable_pages = %zu, "
1116 "contiguous_pages = %zu, mapped at %p\n",
1139 fprintf(stderr,
" pool for blocks of span objects:\n");
1141 fprintf(stderr,
" pool for large object spans:\n");
1144 " pool for size class %zu (object size %hu bytes):\n",
1149 fprintf(stderr,
" fullness class %zu is empty\n",
j);
1154 fprintf(stderr,
" fullness class %zu:\n",
j);
1161 " span descriptor at "
1164 ", pages = %zu, objects free = %hu/%hu\n",
1211 size_t usable_pages;
1213 size_t metadata_bytes;
1218 elog(
ERROR,
"dsa_area space must be at least %zu, but %zu provided",
1230 Assert(metadata_bytes <= size);
1238 memset(place, 0,
sizeof(*control));
1246 control->
handle = control_handle;
1273 segment_map->
segment = control_segment;
1288 if (usable_pages > 0)
1325 segment_map->
segment = segment;
1336 if (control->
refcnt == 0)
1340 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1341 errmsg(
"could not attach to dynamic shared area")));
1379 pool->
spans[1] = span_pointer;
1381 span->
start = start;
1417 span_pointer = pool->
spans[fromclass];
1433 pool->
spans[toclass] = span_pointer;
1487 block = span->
start;
1492 result = block + span->
firstfree * size;
1570 span_pointer = pool->
spans[fclass];
1582 next_span_pointer = span->
nextspan;
1599 if (tfclass < fclass)
1602 if (pool->
spans[fclass] == span_pointer)
1607 if (nextspan != NULL)
1618 if (nextspan != NULL)
1623 pool->
spans[tfclass] = span_pointer;
1635 span_pointer = next_span_pointer;
1679 if (segment_map == NULL)
1682 if (segment_map == NULL)
1695 "dsa_allocate could not find %zu free pages for superblock",
1714 span_pointer = start_pointer;
1718 init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1719 for (
i = 0;
i < npages; ++
i)
1720 segment_map->
pagemap[first_page +
i] = span_pointer;
1757 "dsa_area could not attach to a segment that has been freed");
1760 if (segment == NULL)
1761 elog(
ERROR,
"dsa_area could not attach to segment");
1765 segment_map->
segment = segment;
1857 segment_map->
header = NULL;
1908 pool->
spans[fclass]);
1914 pool->
spans[fclass] = span_pointer;
1999 size_t threshold = (size_t) 1 << (bin - 1);
2008 size_t contiguous_pages;
2011 next_segment_index = segment_map->
header->
next;
2015 if (contiguous_pages >= threshold && contiguous_pages < npages)
2017 segment_index = next_segment_index;
2022 if (contiguous_pages < threshold)
2044 next->header->prev = segment_index;
2054 if (contiguous_pages >= npages)
2058 segment_index = next_segment_index;
2075 size_t metadata_bytes;
2078 size_t usable_pages;
2137 if (requested_pages > usable_pages)
2143 usable_pages = requested_pages;
2166 if (segment == NULL)
2188 segment_map->
segment = segment;
2221 next->header->prev = new_index;
2243 size_t freed_segment_counter;
2279 size_t freed_segment_counter;
#define pg_read_barrier()
elog(ERROR, "%s: %s", p2, msg)
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
static void check_for_freed_segments(dsa_area *area)
dsa_area * dsa_create_in_place(void *place, size_t size, int tranche_id, dsm_segment *segment)
static const uint16 dsa_size_classes[]
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
#define DSA_AREA_LOCK(area)
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE
static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)
#define DSA_SEGMENT_INDEX_NONE
#define DSA_SEGMENT_HEADER_MAGIC
void dsa_trim(dsa_area *area)
#define DSA_SPAN_NOTHING_FREE
#define DSA_MAKE_POINTER(segment_number, offset)
#define get_segment_index(area, segment_map_ptr)
void dsa_on_shmem_exit_release_in_place(int code, Datum place)
#define DSA_INITIAL_SEGMENT_SIZE
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
static dsa_pointer alloc_object(dsa_area *area, int size_class)
#define DSA_PAGES_PER_SUPERBLOCK
#define DSA_SIZE_CLASS_MAP_QUANTUM
static size_t contiguous_pages_to_segment_bin(size_t n)
static const uint8 dsa_size_class_map[]
dsa_pointer dsa_allocate_extended(dsa_area *area, size_t size, int flags)
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
dsa_area * dsa_attach(dsa_handle handle)
#define DSA_SUPERBLOCK_SIZE
#define DsaAreaPoolToDsaPointer(area, p)
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
static void check_for_freed_segments_locked(dsa_area *area)
#define DSA_EXTRACT_OFFSET(dp)
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
#define DSA_SCLASS_LOCK(area, sclass)
void dsa_release_in_place(void *place)
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
void dsa_set_size_limit(dsa_area *area, size_t limit)
#define DSA_SCLASS_BLOCK_OF_SPANS
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
static void unlink_span(dsa_area *area, dsa_area_span *span)
dsa_area * dsa_create(int tranche_id)
#define DSA_SCLASS_SPAN_LARGE
#define DSA_NUM_SIZE_CLASSES
void dsa_unpin(dsa_area *area)
void dsa_pin_mapping(dsa_area *area)
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
#define NextFreeObjectIndex(object)
void dsa_dump(dsa_area *area)
dsa_handle dsa_get_handle(dsa_area *area)
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
void dsa_detach(dsa_area *area)
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
dsa_area * dsa_attach_in_place(void *place, dsm_segment *segment)
#define DSA_MAX_SEGMENT_SIZE
#define DSA_FULLNESS_CLASSES
void dsa_free(dsa_area *area, dsa_pointer dp)
#define DSA_NUM_SEGMENT_BINS
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment)
size_t dsa_minimum_size(void)
void dsa_pin(dsa_area *area)
#define DSA_POINTER_FORMAT
#define InvalidDsaPointer
#define DSA_HANDLE_INVALID
#define DsaPointerIsValid(x)
dsm_handle dsm_segment_handle(dsm_segment *seg)
void * dsm_segment_address(dsm_segment *seg)
void dsm_detach(dsm_segment *seg)
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
dsm_segment * dsm_attach(dsm_handle h)
dsm_segment * dsm_create(Size size, int flags)
void dsm_pin_mapping(dsm_segment *seg)
void dsm_unpin_segment(dsm_handle handle)
void dsm_pin_segment(dsm_segment *seg)
#define DSM_HANDLE_INVALID
int errdetail(const char *fmt,...)
int errcode(int sqlerrcode)
int errmsg(const char *fmt,...)
#define ereport(elevel,...)
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
#define fpm_size_to_pages(sz)
Assert(fmt[strlen(fmt) - 1] !='\n')
bool LWLockHeldByMe(LWLock *lock)
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
void LWLockRelease(LWLock *lock)
void LWLockInitialize(LWLock *lock, int tranche_id)
void pfree(void *pointer)
#define AllocHugeSizeIsValid(size)
#define AllocSizeIsValid(size)
#define pg_leftmost_one_pos_size_t
static Datum PointerGetDatum(const void *X)
static Pointer DatumGetPointer(Datum X)
dsa_segment_header segment_header
size_t total_segment_size
dsa_segment_index high_segment_index
size_t max_total_segment_size
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
size_t freed_segment_counter
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
dsa_pointer spans[DSA_FULLNESS_CLASSES]
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
dsa_segment_index high_segment_index
size_t freed_segment_counter
dsa_area_control * control
dsa_segment_header * header