68 #define DSA_INITIAL_SEGMENT_SIZE ((size_t) (1 * 1024 * 1024))
77 #define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2
84 #if SIZEOF_DSA_POINTER == 4
85 #define DSA_OFFSET_WIDTH 27
87 #define DSA_OFFSET_WIDTH 40
94 #define DSA_MAX_SEGMENTS \
95 Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
98 #define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
101 #define DSA_MAX_SEGMENT_SIZE ((size_t) 1 << DSA_OFFSET_WIDTH)
104 #define DSA_PAGES_PER_SUPERBLOCK 16
111 #define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608
114 #define DSA_MAKE_POINTER(segment_number, offset) \
115 (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
118 #define DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)
121 #define DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)
127 #define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)
133 #define DSA_NUM_SEGMENT_BINS 16
140 #define contiguous_pages_to_segment_bin(n) Min(fls(n), DSA_NUM_SEGMENT_BINS - 1)
143 #define DSA_AREA_LOCK(area) (&area->control->lock)
144 #define DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)
213 #define NextFreeObjectIndex(object) (* (uint16 *) (object))
238 8, 16, 24, 32, 40, 48, 56, 64,
243 1280, 1560, 1816, 2048,
244 2616, 3120, 3640, 4096,
245 5456, 6552, 7280, 8192
247 #define DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)
250 #define DSA_SCLASS_BLOCK_OF_SPANS 0
251 #define DSA_SCLASS_SPAN_LARGE 1
260 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
261 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
262 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
263 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
264 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
265 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
266 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
267 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
269 #define DSA_SIZE_CLASS_MAP_QUANTUM 8
277 #define DSA_FULLNESS_CLASSES 4
329 #define DsaAreaPoolToDsaPointer(area, p) \
330 DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
377 #define DSA_SPAN_NOTHING_FREE ((uint16) -1)
378 #define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
381 #define get_segment_index(area, segment_map_ptr) \
382 (segment_map_ptr - &area->segment_maps[0])
388 int fromclass,
int toclass);
518 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
519 errmsg(
"could not attach to dynamic shared area")));
610 if (--control->
refcnt == 0)
677 elog(
ERROR,
"invalid DSA memory alloc request size %zu", size);
700 (
errcode(ERRCODE_OUT_OF_MEMORY),
702 errdetail(
"Failed on DSA request of size %zu.",
711 if (segment_map == NULL)
713 if (segment_map == NULL)
722 (
errcode(ERRCODE_OUT_OF_MEMORY),
724 errdetail(
"Failed on DSA request of size %zu.",
738 "dsa_allocate could not find %zu free pages", npages);
747 init_span(area, span_pointer, pool, start_pointer, npages,
749 segment_map->
pagemap[first_page] = span_pointer;
756 return start_pointer;
780 uint16 mid = (min + max) / 2;
783 if (class_size < size)
803 (
errcode(ERRCODE_OUT_OF_MEMORY),
805 errdetail(
"Failed on DSA request of size %zu.", size)));
837 span_pointer = segment_map->
pagemap[pageno];
851 #ifdef CLOBBER_FREED_MEMORY
871 #ifdef CLOBBER_FREED_MEMORY
872 memset(
object, 0x7f, size);
878 Assert(
object >= superblock);
880 Assert((
object - superblock) % size == 0);
882 span->
firstfree = (
object - superblock) / size;
1045 span_pointer = pool->
spans[1];
1054 span_pointer =
next;
1078 fprintf(stderr,
" max_total_segment_size: %zu\n",
1080 fprintf(stderr,
" total_segment_size: %zu\n",
1084 fprintf(stderr,
" segment bins:\n");
1092 " segment bin %zu (at least %d contiguous pages free):\n",
1103 " segment index %zu, usable_pages = %zu, "
1104 "contiguous_pages = %zu, mapped at %p\n",
1127 fprintf(stderr,
" pool for blocks of span objects:\n");
1129 fprintf(stderr,
" pool for large object spans:\n");
1132 " pool for size class %zu (object size %hu bytes):\n",
1137 fprintf(stderr,
" fullness class %zu is empty\n",
j);
1142 fprintf(stderr,
" fullness class %zu:\n",
j);
1149 " span descriptor at "
1152 ", pages = %zu, objects free = %hu/%hu\n",
1199 size_t usable_pages;
1201 size_t metadata_bytes;
1206 elog(
ERROR,
"dsa_area space must be at least %zu, but %zu provided",
1218 Assert(metadata_bytes <= size);
1226 memset(place, 0,
sizeof(*control));
1234 control->
handle = control_handle;
1261 segment_map->
segment = control_segment;
1276 if (usable_pages > 0)
1313 segment_map->
segment = segment;
1324 if (control->
refcnt == 0)
1328 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1329 errmsg(
"could not attach to dynamic shared area")));
1367 pool->
spans[1] = span_pointer;
1369 span->
start = start;
1405 span_pointer = pool->
spans[fromclass];
1421 pool->
spans[toclass] = span_pointer;
1475 block = span->
start;
1480 result = block + span->
firstfree * size;
1558 span_pointer = pool->
spans[fclass];
1570 next_span_pointer = span->
nextspan;
1587 if (tfclass < fclass)
1590 if (pool->
spans[fclass] == span_pointer)
1595 if (nextspan != NULL)
1606 if (nextspan != NULL)
1611 pool->
spans[tfclass] = span_pointer;
1623 span_pointer = next_span_pointer;
1667 if (segment_map == NULL)
1670 if (segment_map == NULL)
1683 "dsa_allocate could not find %zu free pages for superblock",
1702 span_pointer = start_pointer;
1706 init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1707 for (
i = 0;
i < npages; ++
i)
1708 segment_map->
pagemap[first_page +
i] = span_pointer;
1745 "dsa_area could not attach to a segment that has been freed");
1748 if (segment == NULL)
1749 elog(
ERROR,
"dsa_area could not attach to segment");
1753 segment_map->
segment = segment;
1845 segment_map->
header = NULL;
1896 pool->
spans[fclass]);
1902 pool->
spans[fclass] = span_pointer;
1987 size_t threshold = (size_t) 1 << (bin - 1);
1996 size_t contiguous_pages;
1999 next_segment_index = segment_map->
header->
next;
2003 if (contiguous_pages >= threshold && contiguous_pages < npages)
2005 segment_index = next_segment_index;
2010 if (contiguous_pages < threshold)
2032 next->header->prev = segment_index;
2042 if (contiguous_pages >= npages)
2046 segment_index = next_segment_index;
2063 size_t metadata_bytes;
2066 size_t usable_pages;
2125 if (requested_pages > usable_pages)
2131 usable_pages = requested_pages;
2154 if (segment == NULL)
2176 segment_map->
segment = segment;
2209 next->header->prev = new_index;
2231 size_t freed_segment_counter;
2267 size_t freed_segment_counter;
#define pg_read_barrier()
elog(ERROR, "%s: %s", p2, msg)
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
static void check_for_freed_segments(dsa_area *area)
dsa_area * dsa_create_in_place(void *place, size_t size, int tranche_id, dsm_segment *segment)
static const uint16 dsa_size_classes[]
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
#define DSA_AREA_LOCK(area)
#define contiguous_pages_to_segment_bin(n)
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE
static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)
#define DSA_SEGMENT_INDEX_NONE
#define DSA_SEGMENT_HEADER_MAGIC
void dsa_trim(dsa_area *area)
#define DSA_SPAN_NOTHING_FREE
#define DSA_MAKE_POINTER(segment_number, offset)
#define get_segment_index(area, segment_map_ptr)
void dsa_on_shmem_exit_release_in_place(int code, Datum place)
#define DSA_INITIAL_SEGMENT_SIZE
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
static dsa_pointer alloc_object(dsa_area *area, int size_class)
#define DSA_PAGES_PER_SUPERBLOCK
#define DSA_SIZE_CLASS_MAP_QUANTUM
static const uint8 dsa_size_class_map[]
dsa_pointer dsa_allocate_extended(dsa_area *area, size_t size, int flags)
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
dsa_area * dsa_attach(dsa_handle handle)
#define DSA_SUPERBLOCK_SIZE
#define DsaAreaPoolToDsaPointer(area, p)
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
static void check_for_freed_segments_locked(dsa_area *area)
#define DSA_EXTRACT_OFFSET(dp)
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
#define DSA_SCLASS_LOCK(area, sclass)
void dsa_release_in_place(void *place)
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
void dsa_set_size_limit(dsa_area *area, size_t limit)
#define DSA_SCLASS_BLOCK_OF_SPANS
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
static void unlink_span(dsa_area *area, dsa_area_span *span)
dsa_area * dsa_create(int tranche_id)
#define DSA_SCLASS_SPAN_LARGE
#define DSA_NUM_SIZE_CLASSES
void dsa_unpin(dsa_area *area)
void dsa_pin_mapping(dsa_area *area)
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
#define NextFreeObjectIndex(object)
void dsa_dump(dsa_area *area)
dsa_handle dsa_get_handle(dsa_area *area)
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
void dsa_detach(dsa_area *area)
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
dsa_area * dsa_attach_in_place(void *place, dsm_segment *segment)
#define DSA_MAX_SEGMENT_SIZE
#define DSA_FULLNESS_CLASSES
void dsa_free(dsa_area *area, dsa_pointer dp)
#define DSA_NUM_SEGMENT_BINS
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment)
size_t dsa_minimum_size(void)
void dsa_pin(dsa_area *area)
#define DSA_POINTER_FORMAT
#define InvalidDsaPointer
#define DsaPointerIsValid(x)
dsm_handle dsm_segment_handle(dsm_segment *seg)
void * dsm_segment_address(dsm_segment *seg)
void dsm_detach(dsm_segment *seg)
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
dsm_segment * dsm_attach(dsm_handle h)
dsm_segment * dsm_create(Size size, int flags)
void dsm_pin_mapping(dsm_segment *seg)
void dsm_unpin_segment(dsm_handle handle)
void dsm_pin_segment(dsm_segment *seg)
#define DSM_HANDLE_INVALID
int errdetail(const char *fmt,...)
int errcode(int sqlerrcode)
int errmsg(const char *fmt,...)
#define ereport(elevel,...)
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
#define fpm_size_to_pages(sz)
Assert(fmt[strlen(fmt) - 1] !='\n')
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
void LWLockRelease(LWLock *lock)
void LWLockInitialize(LWLock *lock, int tranche_id)
bool LWLockHeldByMe(LWLock *l)
void pfree(void *pointer)
#define AllocHugeSizeIsValid(size)
#define AllocSizeIsValid(size)
#define DatumGetPointer(X)
#define PointerGetDatum(X)
dsa_segment_header segment_header
size_t total_segment_size
dsa_segment_index high_segment_index
size_t max_total_segment_size
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
size_t freed_segment_counter
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
dsa_pointer spans[DSA_FULLNESS_CLASSES]
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
dsa_segment_index high_segment_index
size_t freed_segment_counter
dsa_area_control * control
dsa_segment_header * header